Repository: wangheda/youtube-8m
Branch: master
Commit: 07e54b387ee0
Files: 581
Total size: 3.4 MB
Directory structure:
gitextract_1ok4slma/
├── .gitignore
├── .gitmodules
├── LICENSE
├── README.md
├── eda/
│ ├── explore.ipynb
│ └── vertical.tsv
├── youtube-8m-ensemble/
│ ├── .vimrc
│ ├── CONTRIBUTING.md
│ ├── LICENSE
│ ├── README.md
│ ├── __init__.py
│ ├── all_ensemble_models/
│ │ ├── .vimrc
│ │ ├── __init__.py
│ │ ├── attention_linear_model.py
│ │ ├── attention_linmatrix_model.py
│ │ ├── attention_matrix_model.py
│ │ ├── attention_moe_matrix_model.py
│ │ ├── attention_moe_model.py
│ │ ├── attention_rectified_linear_model.py
│ │ ├── deep_combine_chain_model.py
│ │ ├── input_moe_model.py
│ │ ├── linear_regression_model.py
│ │ ├── logistic_model.py
│ │ ├── matrix_regression_model.py
│ │ ├── mean_model.py
│ │ ├── moe_model.py
│ │ └── nonunit_matrix_regression_model.py
│ ├── average_precision_calculator.py
│ ├── check_distillation.py
│ ├── check_video_id.py
│ ├── check_video_id_match.py
│ ├── cloudml-gpu-distributed.yaml
│ ├── cloudml-gpu.yaml
│ ├── data_augmentation.py
│ ├── ensemble_command.example
│ ├── ensemble_level_models.py
│ ├── ensemble_scripts/
│ │ ├── .vimrc
│ │ ├── after_submission_no1.conf
│ │ ├── after_submission_no2.conf
│ │ ├── after_submission_no3.conf
│ │ ├── after_submission_no4.conf
│ │ ├── auto-preensemble-deep_combine_chain_model.sh
│ │ ├── auto-preensemble-matrix_model.sh
│ │ ├── check-video_id.sh
│ │ ├── check-video_id_match.sh
│ │ ├── combine-tfrecords-frame-v2.sh
│ │ ├── combine-tfrecords-frame.sh
│ │ ├── combine-tfrecords-video-v2.sh
│ │ ├── combine-tfrecords-video.sh
│ │ ├── ensemble_no1.conf
│ │ ├── ensemble_no10.conf
│ │ ├── ensemble_no11.conf
│ │ ├── ensemble_no12.conf
│ │ ├── ensemble_no13.conf
│ │ ├── ensemble_no14.conf
│ │ ├── ensemble_no15.conf
│ │ ├── ensemble_no16.conf
│ │ ├── ensemble_no17.conf
│ │ ├── ensemble_no18.conf
│ │ ├── ensemble_no19.conf
│ │ ├── ensemble_no2.conf
│ │ ├── ensemble_no20.conf
│ │ ├── ensemble_no21.conf
│ │ ├── ensemble_no3.conf
│ │ ├── ensemble_no4.conf
│ │ ├── ensemble_no5.conf
│ │ ├── ensemble_no6.conf
│ │ ├── ensemble_no7.conf
│ │ ├── ensemble_no8.conf
│ │ ├── ensemble_no9.conf
│ │ ├── eval-attention_linear_model.sh
│ │ ├── eval-attention_linmatrix_model.sh
│ │ ├── eval-attention_matrix_model.sh
│ │ ├── eval-attention_moe_matrix_model.sh
│ │ ├── eval-attention_moe_model.sh
│ │ ├── eval-attention_rectified_linear_model.sh
│ │ ├── eval-deep_combine_chain_model.sh
│ │ ├── eval-input_moe_model.sh
│ │ ├── eval-linear_model.sh
│ │ ├── eval-matrix_model.sh
│ │ ├── eval-mean_model.sh
│ │ ├── eval-moe_model.sh
│ │ ├── eval-nonunit_matrix_model.sh
│ │ ├── explore-mean_model.log
│ │ ├── explore-mean_model.sh
│ │ ├── final_submission.conf
│ │ ├── infer-attention_linear_model.sh
│ │ ├── infer-attention_linmatrix_model.sh
│ │ ├── infer-attention_matrix_model.sh
│ │ ├── infer-attention_moe_matrix_model.sh
│ │ ├── infer-attention_moe_model.sh
│ │ ├── infer-attention_rectified_linear_model.sh
│ │ ├── infer-linear_model.sh
│ │ ├── infer-matrix_model.sh
│ │ ├── infer-mean_model.sh
│ │ ├── infer-moe_model.sh
│ │ ├── make-bagging-of-ensembles.sh
│ │ ├── make-virtual-groups.sh
│ │ ├── preensemble-attention_matrix_model.sh
│ │ ├── preensemble-matrix_model.sh
│ │ ├── preensemble-mean_model.sh
│ │ ├── train-attention_linear_model.sh
│ │ ├── train-attention_linmatrix_model.sh
│ │ ├── train-attention_matrix_model.sh
│ │ ├── train-attention_moe_matrix_model.sh
│ │ ├── train-attention_moe_model.sh
│ │ ├── train-attention_rectified_linear_model.sh
│ │ ├── train-deep_combine_chain_model.sh
│ │ ├── train-input_moe_model.sh
│ │ ├── train-linear_model.sh
│ │ ├── train-matrix_model.sh
│ │ ├── train-matrix_model_lr.sh
│ │ ├── train-mean_model.sh
│ │ ├── train-moe_model.sh
│ │ └── train-nonunit_matrix_model.sh
│ ├── eval.py
│ ├── eval_util.py
│ ├── feature_transform.py
│ ├── inference-combine-tfrecords-frame.py
│ ├── inference-combine-tfrecords-video.py
│ ├── inference-pre-ensemble.py
│ ├── inference.py
│ ├── losses.py
│ ├── mean_average_precision_calculator.py
│ ├── model_selection_scripts/
│ │ ├── .vimrc
│ │ ├── extend-step-mean_model.sh
│ │ ├── get_extend_candidates.py
│ │ ├── get_patterns.py
│ │ ├── get_top_k.py
│ │ └── greedy-selection-mean_model.sh
│ ├── model_utils.py
│ ├── models.py
│ ├── readers.py
│ ├── top_k_scripts/
│ │ ├── eval-attention_matrix_model.sh
│ │ ├── infer-attention_matrix_model.sh
│ │ ├── preensemble-attention_matrix_model.sh
│ │ ├── run_top_k.sh
│ │ └── train-attention_matrix_model.sh
│ ├── train.py
│ ├── training_utils/
│ │ ├── del.py
│ │ ├── sample_conf.py
│ │ ├── sample_freq.py
│ │ └── select.py
│ └── utils.py
├── youtube-8m-wangheda/
│ ├── .vimrc
│ ├── CONTRIBUTING.md
│ ├── LICENSE
│ ├── README.md
│ ├── __init__.py
│ ├── all_data_augmentation/
│ │ ├── __init__.py
│ │ ├── clipping_augmenter.py
│ │ ├── default_augmenter.py
│ │ ├── half_augmenter.py
│ │ ├── half_video_augmenter.py
│ │ └── noise_augmenter.py
│ ├── all_feature_transform/
│ │ ├── __init__.py
│ │ ├── avg_transformer.py
│ │ ├── default_transformer.py
│ │ ├── engineer_transformer.py
│ │ ├── identical_transformer.py
│ │ └── resolution_transformer.py
│ ├── all_frame_models/
│ │ ├── .vimrc
│ │ ├── __init__.py
│ │ ├── bilstm_model.py
│ │ ├── biunilstm_model.py
│ │ ├── cnn_deep_combine_chain_model.py
│ │ ├── cnn_kmax_model.py
│ │ ├── cnn_lstm_memory_model.py
│ │ ├── cnn_lstm_memory_multitask_model.py
│ │ ├── cnn_lstm_memory_normalization_model.py
│ │ ├── cnn_model.py
│ │ ├── dbof_model.py
│ │ ├── deep_cnn_deep_combine_chain_model.py
│ │ ├── deep_lstm_model.py
│ │ ├── distillchain_cnn_deep_combine_chain_model.py
│ │ ├── distillchain_lstm_attention_max_pooling_model.py
│ │ ├── distillchain_lstm_cnn_deep_combine_chain_model.py
│ │ ├── distillchain_lstm_memory_deep_combine_chain_model.py
│ │ ├── distillchain_lstm_parallel_finaloutput_model.py
│ │ ├── distillchain_multiscale_cnn_lstm_model.py
│ │ ├── frame_seg_model.py
│ │ ├── framehop_lstm_memory_deep_combine_chain_model.py
│ │ ├── framehop_lstm_memory_model.py
│ │ ├── gru_pooling_model.py
│ │ ├── gru_with_pooling_model.py
│ │ ├── layernorm_lstm_memory_model.py
│ │ ├── logistic_model.py
│ │ ├── lstm_advanced_model.py
│ │ ├── lstm_attention_lstm_model.py
│ │ ├── lstm_attention_max_pooling_model.py
│ │ ├── lstm_attention_model.py
│ │ ├── lstm_auxloss_deep_combine_chain_model.py
│ │ ├── lstm_cnn_deep_combine_chain_model.py
│ │ ├── lstm_divided_model.py
│ │ ├── lstm_look_back_model.py
│ │ ├── lstm_memory_chain_model.py
│ │ ├── lstm_memory_deep_chain_model.py
│ │ ├── lstm_memory_input_chain_model.py
│ │ ├── lstm_memory_model.py
│ │ ├── lstm_memory_multitask_model.py
│ │ ├── lstm_memory_normalization_model.py
│ │ ├── lstm_memory_parallel_chain_model.py
│ │ ├── lstm_model.py
│ │ ├── lstm_multi_attention_model.py
│ │ ├── lstm_multi_pooling_model.py
│ │ ├── lstm_parallel_finaloutput_model.py
│ │ ├── lstm_parallel_memory_model.py
│ │ ├── lstm_parallel_model.py
│ │ ├── lstm_pooling_model.py
│ │ ├── lstm_positional_attention_max_pooling_model.py
│ │ ├── lstm_with_mean_input_model.py
│ │ ├── lstm_with_pooling_model.py
│ │ ├── mm_lstm_memory_model.py
│ │ ├── multi_view_cnn_deep_combine_chain_model.py
│ │ ├── multires_lstm_memory_deep_combine_chain_model.py
│ │ ├── multiscale_cnn_lstm_model.py
│ │ ├── positional_cnn_deep_combine_chain_model.py
│ │ ├── progressive_attention_lstm_model.py
│ │ └── wide_and_deep_model.py
│ ├── all_video_models/
│ │ ├── .vimrc
│ │ ├── __init__.py
│ │ ├── chain_main_relu_moe_model.py
│ │ ├── chain_moe_model.py
│ │ ├── chain_support_relu_moe_model.py
│ │ ├── deep_chain_model.py
│ │ ├── deep_combine_chain_model.py
│ │ ├── distillchain_deep_combine_chain_model.py
│ │ ├── hidden_chain_model.py
│ │ ├── hidden_combine_chain_model.py
│ │ ├── logistic_model.py
│ │ ├── mlp_moe_model.py
│ │ ├── moe_model.py
│ │ ├── multitask_divergence_deep_combine_chain_model.py
│ │ ├── multitask_divergence_moe_model.py
│ │ ├── multitask_moe_model.py
│ │ ├── shortcut_chain_support_relu_moe_model.py
│ │ └── stage2_logistic_model.py
│ ├── average_precision_calculator.py
│ ├── bagging_scripts/
│ │ ├── cnn-deep-combine-chain-bagging.sh
│ │ ├── distillation-video-dcc-bagging.sh
│ │ ├── lstmattention8max-bagging.sh
│ │ ├── lstmparalleloutput-bagging.sh
│ │ └── video-deep-combine-chain-bagging.sh
│ ├── boosting_scripts/
│ │ ├── cnn-deep-combine-chain-boosting.sh
│ │ ├── distillation-cnn-dcc-boosting.sh
│ │ ├── distillation-lstmcnn-dcc-boosting.sh
│ │ ├── distillation-lstmparalleloutput-boosting.sh
│ │ ├── distillation-multilstm-dcc-boosting.sh
│ │ ├── distillation-multiscale-cnn-lstm-boosting.sh
│ │ ├── distillation-positional-lstmattention8max-boosting.sh
│ │ ├── distillation-video-dcc-boosting.sh
│ │ ├── lstmattention8max-boosting-weightclip.sh
│ │ ├── lstmparalleloutput-boosting-weightclip.sh
│ │ ├── video-deep-combine-chain-boosting-discardhopeless.sh
│ │ ├── video-deep-combine-chain-boosting-weightclip.sh
│ │ └── video-deep-combine-chain-boosting.sh
│ ├── cascade_scripts/
│ │ ├── distillchain-v2-hybridchain.sh
│ │ ├── distillchain-v2-hybridchain2.sh
│ │ └── distillchain-v2-videochain.sh
│ ├── cloudml-gpu-distributed.yaml
│ ├── cloudml-gpu.yaml
│ ├── data_augmentation.py
│ ├── data_augmentation_scripts/
│ │ ├── eval-chaining-video.sh
│ │ ├── run-chaining-cnn.sh
│ │ ├── run-chaining-lstm.sh
│ │ ├── run-chaining-video.sh
│ │ ├── run-multiple-attention-pooling-positional-embedding.sh
│ │ ├── run-multiscale-cnn-lstm-model.sh
│ │ └── run-parallel-lstm-memory.sh
│ ├── eval.py
│ ├── eval.sh
│ ├── eval_scripts/
│ │ ├── eval-att-lstm.sh
│ │ ├── eval-att.sh
│ │ ├── eval-bi-uni-lstm.sh
│ │ ├── eval-chain-model-relu.sh
│ │ ├── eval-chain-model-suprelu.sh
│ │ ├── eval-chain-moe-0.4.sh
│ │ ├── eval-chain-moe-freq.sh
│ │ ├── eval-chain-moe-suprelu-vert+freq.sh
│ │ ├── eval-chain-moe.sh
│ │ ├── eval-cnn-deep-combine-chain.sh
│ │ ├── eval-cnn-model.sh
│ │ ├── eval-dbof.sh
│ │ ├── eval-deep-cnn-deep-combine-chain.sh
│ │ ├── eval-distill-video-dcc-noise-scene1.sh
│ │ ├── eval-distill-video-dcc-noise-scene2.sh
│ │ ├── eval-distillchain-cnn-dcc.sh
│ │ ├── eval-distillchain-lstmcnn.sh
│ │ ├── eval-distillchain-lstmparalleloutput.sh
│ │ ├── eval-distillchain-multilstm.sh
│ │ ├── eval-distillchain-v2-boostinglstmparalleloutput.sh
│ │ ├── eval-distillchain-v2-lstmattention8max.sh
│ │ ├── eval-distillchain-v2-lstmcnn.sh
│ │ ├── eval-distillchain-v2-lstmparalleloutput.sh
│ │ ├── eval-distillchain-v2-multilstm.sh
│ │ ├── eval-distillchain-v2-multiscale-cnnlstm.sh
│ │ ├── eval-distillchain-v2-video-dcc.sh
│ │ ├── eval-distillchain-video-dcc.sh
│ │ ├── eval-frame-seg.sh
│ │ ├── eval-framehop-lstmmem.sh
│ │ ├── eval-layer-chain-moe8-freq.sh
│ │ ├── eval-layer-moe-vert.sh
│ │ ├── eval-lstm-attention-8max.sh
│ │ ├── eval-lstm-cnn-deep-combine-chain.sh
│ │ ├── eval-lstm-look-back.sh
│ │ ├── eval-lstm-positional-attention-8max.sh
│ │ ├── eval-lstmmem-augmenter.sh
│ │ ├── eval-lstmmem-chain-freq.sh
│ │ ├── eval-lstmmem-chain.sh
│ │ ├── eval-lstmmem-cnnlstm.sh
│ │ ├── eval-lstmmem-deep-chain.sh
│ │ ├── eval-lstmmem-deep-combine-chain-length.sh
│ │ ├── eval-lstmmem-dropout.sh
│ │ ├── eval-lstmmem-feature.sh
│ │ ├── eval-lstmmem-input-chain.sh
│ │ ├── eval-lstmmem-input-noise.sh
│ │ ├── eval-lstmmem-l2norm.sh
│ │ ├── eval-lstmmem-layernorm.sh
│ │ ├── eval-lstmmem-lowres.sh
│ │ ├── eval-lstmmem-no-transform.sh
│ │ ├── eval-lstmmem-noise.sh
│ │ ├── eval-lstmmem-parallel.sh
│ │ ├── eval-lstmmem-shortcut-chain-freq.sh
│ │ ├── eval-lstmmem2048.sh
│ │ ├── eval-lstmmemory-audio.sh
│ │ ├── eval-lstmmemory-layer1.sh
│ │ ├── eval-lstmmemory.sh
│ │ ├── eval-lstmoutput-parallel.sh
│ │ ├── eval-mem.sh
│ │ ├── eval-mm-lstm.sh
│ │ ├── eval-moe-baseline.sh
│ │ ├── eval-moe-batchagreement1.sh
│ │ ├── eval-moe-batchagreement2.sh
│ │ ├── eval-moe-batchagreement3.sh
│ │ ├── eval-moe-model.sh
│ │ ├── eval-moe-topk-batchagreement1.sh
│ │ ├── eval-moe-topk-batchagreement2.sh
│ │ ├── eval-moe-topk-batchagreement3.sh
│ │ ├── eval-multi-lstmmem-deep-chain.sh
│ │ ├── eval-multi-view-cnn-deep-combine-chain.sh
│ │ ├── eval-multires-lstm-deep-combine-chain.sh
│ │ ├── eval-multitask-ce.sh
│ │ ├── eval-multitask.sh
│ │ ├── eval-positional-cnn-dcc.sh
│ │ ├── eval-stage2-logistic.sh
│ │ ├── eval-stage2-moe.sh
│ │ ├── eval-video-deep-chain.sh
│ │ ├── eval-video-deep-combine-addnoise.sh
│ │ ├── eval-video-deep-combine-chain-dropout.sh
│ │ ├── eval-video-deep-combine-chain-labelsmoothing.sh
│ │ ├── eval-video-deep-combine-chain-noise.sh
│ │ ├── eval-video-deep-combine-chain.sh
│ │ ├── eval-video-distillchain-video-dcc.sh
│ │ ├── eval-video-divergence-chain-model.sh
│ │ ├── eval-video-divergence-moe-model.sh
│ │ ├── eval-video-hidden-chain.sh
│ │ ├── eval-video-hidden-combine-chain.sh
│ │ ├── eval-video-logistic.sh
│ │ ├── eval-video-moe.sh
│ │ ├── eval-video-pairwise.sh
│ │ └── eval-video-verydeep-combine-chain.sh
│ ├── eval_util.py
│ ├── feature_transform.py
│ ├── frame_level_models.py
│ ├── infer_scripts/
│ │ ├── infer-attentionlstm_moe4.sh
│ │ ├── infer-biunilstm1024_moe8.sh
│ │ ├── infer-cnn_deep_combine_chain.sh
│ │ ├── infer-cnn_lstmmemory1024_moe8.sh
│ │ ├── infer-cnn_model.sh
│ │ ├── infer-dbof.sh
│ │ ├── infer-deep_cnn_deep_combine.sh
│ │ ├── infer-deeplstm1024_layer6_moe4.sh
│ │ ├── infer-distill_video_dcc.sh
│ │ ├── infer-distillation-cnn-dcc.sh
│ │ ├── infer-distillation-lstmattention8max.sh
│ │ ├── infer-distillation-lstmgate.sh
│ │ ├── infer-distillation-video-dcc.sh
│ │ ├── infer-distillation.sh
│ │ ├── infer-distillchain-cnn-dcc.sh
│ │ ├── infer-distillchain-lstmcnn.sh
│ │ ├── infer-distillchain-lstmparalleloutput.sh
│ │ ├── infer-distillchain-v2-boost-lstmparalleloutput.sh
│ │ ├── infer-distillchain-v2-lstmattention8max.sh
│ │ ├── infer-distillchain-v2-lstmcnn.sh
│ │ ├── infer-distillchain-v2-lstmparalleloutput.sh
│ │ ├── infer-distillchain-v2-multilstm.sh
│ │ ├── infer-distillchain-v2-multiscal-cnnlstm.sh
│ │ ├── infer-distillchain-v2-video-dcc.sh
│ │ ├── infer-frame_seg.sh
│ │ ├── infer-framehop_lstm.sh
│ │ ├── infer-lstm_attention8_max.sh
│ │ ├── infer-lstm_cnn_deep_combine_chain.sh
│ │ ├── infer-lstmattlstm1024_moe8.sh
│ │ ├── infer-lstmmemory-audio.sh
│ │ ├── infer-lstmmemory-layer1.sh
│ │ ├── infer-lstmmemory1024_deep_combine_chain_add_length.sh
│ │ ├── infer-lstmmemory1024_moe8.sh
│ │ ├── infer-lstmmemory2048_moe4.sh
│ │ ├── infer-lstmparallelmemory1024_moe8.sh
│ │ ├── infer-lstmparalleloutput1024_moe8.sh
│ │ ├── infer-model_input.sh
│ │ ├── infer-multilstmmemory1024_moe4_deep_chain.sh
│ │ ├── infer-multires_lstm_deep_combine_chain.sh
│ │ ├── infer-positional-lstmattention8max.sh
│ │ ├── infer-video-distillchain-video-dcc.sh
│ │ ├── infer-video_group_moe4_noise0.2_layer4_elu.sh
│ │ ├── infer-video_logistic.sh
│ │ ├── infer-video_moe16_model.sh
│ │ └── infer-video_very_deep_combine_chain.sh
│ ├── inference-layer.py
│ ├── inference-pre-ensemble-get-input.py
│ ├── inference-pre-ensemble-with-predictions.py
│ ├── inference-pre-ensemble.py
│ ├── inference-sample-error-analysis.py
│ ├── inference-sample-error.py
│ ├── inference-stage1.py
│ ├── inference.py
│ ├── losses.py
│ ├── mean_average_precision_calculator.py
│ ├── model_utils.py
│ ├── models.py
│ ├── readers.py
│ ├── train-with-predictions.py
│ ├── train-with-rebuild.py
│ ├── train.py
│ ├── training_scripts/
│ │ ├── run-cascade-75-chaining-cnn.sh
│ │ ├── run-cascade-75-chaining-lstm-cnn.sh
│ │ ├── run-cascade-75-chaining-lstm.sh
│ │ ├── run-cascade-75-chaining-parallel-lstm.sh
│ │ ├── run-cascade-75-chaining-video.sh
│ │ ├── run-cascade-75-multiple-attention-pooling.sh
│ │ ├── run-cascade-76-chaining-cnn.sh
│ │ ├── run-cascade-76-chaining-lstm-cnn.sh
│ │ ├── run-cascade-76-chaining-lstm.sh
│ │ ├── run-cascade-76-chaining-video.sh
│ │ ├── run-cascade-76-multiple-attention-pooling.sh
│ │ ├── run-cascade-76-multiscale-cnn-lstm.sh
│ │ ├── run-cascade-76-parallel-lstm-boosting.sh
│ │ ├── run-cascade-76-parallel-lstm.sh
│ │ ├── run-chaining-cnn.sh
│ │ ├── run-chaining-deep-cnn.sh
│ │ ├── run-chaining-lstm-cnn.sh
│ │ ├── run-chaining-lstm.sh
│ │ ├── run-chaining-multi-resolution-lstm.sh
│ │ ├── run-chaining-shared-lstm.sh
│ │ ├── run-chaining-video.sh
│ │ ├── run-cnn-lstm.sh
│ │ ├── run-cnn-model.sh
│ │ ├── run-lstm-memory-cell1024.sh
│ │ ├── run-lstm-memory-cell2048.sh
│ │ ├── run-multiple-attention-pooling-positional-embedding.sh
│ │ ├── run-multiscale-cnn-lstm-model.sh
│ │ ├── run-parallel-lstm-memory.sh
│ │ ├── run-parallel-lstm-output.sh
│ │ └── run-temporal-pooling-lstm.sh
│ ├── training_utils/
│ │ ├── del.py
│ │ ├── human_readable_error_analysis.py
│ │ ├── reweight_sample_freq.py
│ │ ├── sample_freq.py
│ │ ├── select.py
│ │ └── video_original_boosting_error_analysis.py
│ ├── utils.py
│ └── video_level_models.py
└── youtube-8m-zhangteng/
├── CONTRIBUTING.md
├── LICENSE
├── README.md
├── YM_framemean.py
├── YM_labels_matrix.py
├── YM_labels_model.py
├── YM_labels_vocab.py
├── YM_readframefeature.py
├── __init__.py
├── average_precision_calculator.py
├── cloudml-gpu-distributed.yaml
├── cloudml-gpu.yaml
├── eval.py
├── eval_autoencoder.py
├── eval_distill.py
├── eval_embedding.py
├── eval_scripts/
│ ├── eval-distillchain_cnndcc_layer2moe4.sh
│ ├── eval-distillchain_lstm_extend_moe8.sh
│ ├── eval-distillchain_lstm_gate_moe8.sh
│ ├── eval-distillchain_lstm_gate_moe8_v2.sh
│ ├── eval-distillchain_lstm_glu2_moe8_v2.sh
│ ├── eval-distillchain_lstm_moe8.sh
│ ├── eval-distillchain_lstm_moe8_v2.sh
│ ├── eval-distillchain_lstm_multiscale2layer_moe8.sh
│ ├── eval-distillchain_lstm_multiscale4layer_moe8.sh
│ ├── eval-distillchain_video_norm_moe8_local.sh
│ ├── eval-distillsplit_lstm_gate_moe8.sh
│ ├── eval-lstm2_attention8_max.sh
│ ├── eval-lstm_attention8_max.sh
│ ├── eval-lstm_gate_multiscale4_moe4.sh
│ ├── eval-lstm_multiscale4_moe4.sh
│ ├── eval-lstm_random_moe8.sh
│ ├── eval-lstm_shortlayers_moe8.sh
│ ├── eval-lstmbiglu_1024_moe8.sh
│ ├── eval-lstmgate1024_moe8.sh
│ ├── eval-lstmglu2_1024_moe8.sh
│ ├── eval-video_knowledge_combine_chain.sh
│ ├── eval-video_notzero_combine_chain.sh
│ ├── eval-video_relabel_combine_chain.sh
│ └── eval-video_softmax_combine_chain.sh
├── eval_util.py
├── frame_level_models.py
├── infer_scripts/
│ ├── infer-distillchain_cnndcc_layer2moe4.sh
│ ├── infer-distillchain_cnndcc_layer2moe4_ensemble.sh
│ ├── infer-distillchain_lstm_extend_moe8.sh
│ ├── infer-distillchain_lstm_gate_moe8.sh
│ ├── infer-distillchain_lstm_gate_moe8_v2.sh
│ ├── infer-distillchain_lstm_glu2_moe8_v2.sh
│ ├── infer-distillchain_lstm_moe8.sh
│ ├── infer-distillchain_lstm_moe8_v2.sh
│ ├── infer-distillchain_lstm_multiscale2layer_moe8.sh
│ ├── infer-distillchain_lstm_multiscale4layer_moe8.sh
│ ├── infer-distillchain_video_norm_moe8.sh
│ ├── infer-distillchain_video_norm_moe8_local.sh
│ ├── infer-distillsplit_lstm_gate_moe8.sh
│ ├── infer-lstm2_attention8_max.sh
│ ├── infer-lstm_attention8_max.sh
│ ├── infer-lstm_attention_max_mean.sh
│ ├── infer-lstm_gate_multiscale4_moe4.sh
│ ├── infer-lstm_multiscale4_moe4.sh
│ ├── infer-lstm_random_mean_moe8.sh
│ ├── infer-lstm_shortlayers_moe8.sh
│ ├── infer-lstmbiglu_1024_moe8.sh
│ ├── infer-lstmgate1024_moe8.sh
│ ├── infer-lstmglu2_1024_moe8.sh
│ ├── infer-video_notzero_combine_chain.sh
│ └── infer-video_relabel_combine_chain.sh
├── inference-pre-ensemble-distill.py
├── inference-pre-ensemble.py
├── inference.py
├── inference_autoencoder.py
├── inference_embedding.py
├── inference_test.py
├── inference_with_rebuild.py
├── labels_autoencoder.py
├── labels_embedding.py
├── labels_rbm.py
├── losses.py
├── losses_embedding.py
├── mean_average_precision_calculator.py
├── model_utils.py
├── models.py
├── readers.py
├── rnn_residual.py
├── train-with-rebuild.py
├── train.py
├── train_autoencoder.py
├── train_embedding.py
├── train_ensemble.py
├── train_scripts/
│ ├── run-attention-pooling-lstm-a.sh
│ ├── run-attention-pooling-lstm-s.sh
│ ├── run-attention-pooling-lstm2lstm.sh
│ ├── run-attention-pooling.sh
│ ├── run-bilstm-a.sh
│ ├── run-cascade-76-lstm-a.sh
│ ├── run-cascade-76-lstm-s.sh
│ ├── run-cascade-76-lstm.sh
│ ├── run-cascade-attention-pooling.sh
│ ├── run-cascade-chaining-cnn-layer2.sh
│ ├── run-cascade-chaining-video-normalize.sh
│ ├── run-cascade-lstm-s-split.sh
│ ├── run-cascade-lstm-s.sh
│ ├── run-cascade-lstm.sh
│ ├── run-cascade-multiscale-cnn-lstm-laery4.sh
│ ├── run-cascade-multiscale-cnn-lstm-layer2.sh
│ ├── run-chaining-video-add-confident.sh
│ ├── run-chaining-video-infrequent-softmax.sh
│ ├── run-chaining-video-normal.sh
│ ├── run-chaining-video-vertical.sh
│ ├── run-lstm-random-augmentation.sh
│ ├── run-lstm-s.sh
│ ├── run-multiscale-cnn-lstm.sh
│ └── run-temporal-segment-lstm.sh
├── training_utils/
│ ├── del.py
│ └── select.py
├── utils.py
├── video_level_models.py
└── writers.py
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
model/
prediction/
*.pdf
*/__pycache__/
*.out
*.csv
*.tfrecord
*.pyc
.*.swp
eda/data/
================================================
FILE: .gitmodules
================================================
[submodule "3rd_party/annoy"]
path = 3rd_party/annoy
url = git@github.com:spotify/annoy.git
[submodule "tensorflow"]
path = tensorflow
url = git@github.com:tensorflow/tensorflow.git
================================================
FILE: LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2017 Heda Wang and Teng Zhang
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: README.md
================================================
# The Monkeytyping Solution to the Youtube-8M Video Understanding Challenge
This is the solution repository of the 2nd place team monkeytyping, licensed under the Apache License 2.0.
## Dependencies
Python 2.7
Tensorflow 1.0
Numpy 1.12
GNU Bash
## Resources
For an understanding of our system, read the report of our solution:
> https://arxiv.org/abs/1706.05150
Our source code:
> https://github.com/wangheda/youtube-8m
## Useful scripts
Training scripts (training a model may take 3-5 days) are in
> youtube-8m-wangheda/training_scripts
> youtube-8m-zhangteng/train_scripts
Eval scripts for selecting best performing checkpoints
> youtube-8m-wangheda/eval_scripts
> youtube-8m-zhangteng/eval_scripts
Infer scripts for generating intermediate files used by ensemble scripts
> youtube-8m-wangheda/infer_scripts
> youtube-8m-zhangteng/infer_scripts
Ensemble scripts
> youtube-8m-ensemble/ensemble_scripts
## Paths of models and data
There are some conventions that we use in our code:
models are saved in
> ./model
train1 data is saved in
> /Youtube-8M/data/frame/train
> /Youtube-8M/data/video/train
validate1 data is saved in
> /Youtube-8M/data/frame/validate
> /Youtube-8M/data/video/validate
test data is saved in
> /Youtube-8M/data/frame/test
> /Youtube-8M/data/video/test
train2 data is saved in
> /Youtube-8M/data/frame/ensemble_train
> /Youtube-8M/data/video/ensemble_train
validate2 data is saved in
> /Youtube-8M/data/frame/ensemble_validate
> /Youtube-8M/data/video/ensemble_validate
intermediate results are stored in
> /Youtube-8M/model_predictions/ensemble_train/[method]
> /Youtube-8M/model_predictions/ensemble_validate/[method]
> /Youtube-8M/model_predictions/test/[method]
## How to generate a solution
### Single model
1. Train a single model
2. evaluate the checkpoints to get the best one
3. infer the checkpoint to get intermediate result.
### Ensemble model
1. Write a configuration file
2. train a stacking model
3. evaluate the stacking model and pick the best checkpoint
4. infer the checkpoint to get a submission file
## Note
Some of the single models are developed by Heda and some by Teng, so they are distributed in two folders.
Bagging models are in `youtube-8m-wangheda/bagging_scripts`.
Boosting and distillation models are in `youtube-8m-wangheda/bagging_scripts`.
Cascade models are in `youtube-8m-wangheda/cascade_scripts`.
Stacking models are in `youtube-8m-ensemble/ensemble_scripts`.
================================================
FILE: eda/explore.ipynb
================================================
{
"cells": [
{
"cell_type": "code",
"execution_count": 16,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import pandas as pd\n",
"import matplotlib.pyplot as plt"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"label_names = pd.read_csv(\"data/label_names.csv\")\n",
"vocabulary = pd.read_csv(\"data/vocabulary.csv\")\n",
"train_labels = pd.read_csv(\"data/train_labels.csv\", header=None, names=[\"id\", \"labels\"])\n",
"validate_labels = pd.read_csv(\"data/validate_labels.csv\", header=None, names=[\"id\", \"labels\"])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Lable names"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/html": [
"
\n",
"
\n",
" \n",
" \n",
" | \n",
" label_id | \n",
" label_name | \n",
"
\n",
" \n",
" \n",
" \n",
" | 0 | \n",
" 0 | \n",
" Games | \n",
"
\n",
" \n",
" | 1 | \n",
" 1 | \n",
" Vehicle | \n",
"
\n",
" \n",
" | 2 | \n",
" 2 | \n",
" Video game | \n",
"
\n",
" \n",
" | 3 | \n",
" 3 | \n",
" Concert | \n",
"
\n",
" \n",
" | 4 | \n",
" 4 | \n",
" Car | \n",
"
\n",
" \n",
" | 5 | \n",
" 5 | \n",
" Dance | \n",
"
\n",
" \n",
" | 6 | \n",
" 6 | \n",
" Animation | \n",
"
\n",
" \n",
" | 7 | \n",
" 7 | \n",
" Musician | \n",
"
\n",
" \n",
" | 8 | \n",
" 8 | \n",
" Football | \n",
"
\n",
" \n",
" | 9 | \n",
" 9 | \n",
" Music video | \n",
"
\n",
" \n",
" | 10 | \n",
" 10 | \n",
" Animal | \n",
"
\n",
" \n",
" | 11 | \n",
" 11 | \n",
" Motorsport | \n",
"
\n",
" \n",
" | 12 | \n",
" 12 | \n",
" Food | \n",
"
\n",
" \n",
" | 13 | \n",
" 13 | \n",
" Musical ensemble | \n",
"
\n",
" \n",
" | 14 | \n",
" 14 | \n",
" Guitar | \n",
"
\n",
" \n",
" | 15 | \n",
" 15 | \n",
" Cartoon | \n",
"
\n",
" \n",
" | 16 | \n",
" 16 | \n",
" Performance art | \n",
"
\n",
" \n",
" | 17 | \n",
" 17 | \n",
" Racing | \n",
"
\n",
" \n",
" | 18 | \n",
" 18 | \n",
" Outdoor recreation | \n",
"
\n",
" \n",
" | 19 | \n",
" 19 | \n",
" PC game | \n",
"
\n",
" \n",
" | 20 | \n",
" 20 | \n",
" Trailer | \n",
"
\n",
" \n",
" | 21 | \n",
" 21 | \n",
" Stadium | \n",
"
\n",
" \n",
" | 22 | \n",
" 22 | \n",
" Nature | \n",
"
\n",
" \n",
" | 23 | \n",
" 23 | \n",
" Mobile phone | \n",
"
\n",
" \n",
" | 24 | \n",
" 24 | \n",
" String instrument | \n",
"
\n",
" \n",
" | 25 | \n",
" 25 | \n",
" Toy | \n",
"
\n",
" \n",
" | 26 | \n",
" 26 | \n",
" Cooking | \n",
"
\n",
" \n",
" | 27 | \n",
" 27 | \n",
" Motorcycle | \n",
"
\n",
" \n",
" | 28 | \n",
" 28 | \n",
" Fashion | \n",
"
\n",
" \n",
" | 29 | \n",
" 29 | \n",
" Smartphone | \n",
"
\n",
" \n",
" | ... | \n",
" ... | \n",
" ... | \n",
"
\n",
" \n",
" | 4686 | \n",
" 4686 | \n",
" Nisekoi | \n",
"
\n",
" \n",
" | 4687 | \n",
" 4687 | \n",
" Hollywood Palladium | \n",
"
\n",
" \n",
" | 4688 | \n",
" 4688 | \n",
" Skullgirls | \n",
"
\n",
" \n",
" | 4689 | \n",
" 4689 | \n",
" Al-Masjid an-Nabawi | \n",
"
\n",
" \n",
" | 4690 | \n",
" 4690 | \n",
" Shadow Fighter | \n",
"
\n",
" \n",
" | 4691 | \n",
" 4691 | \n",
" Reptile | \n",
"
\n",
" \n",
" | 4692 | \n",
" 4692 | \n",
" Abercrombie & Fitch | \n",
"
\n",
" \n",
" | 4693 | \n",
" 4693 | \n",
" Daffy Duck | \n",
"
\n",
" \n",
" | 4694 | \n",
" 4694 | \n",
" Cruelty to animals | \n",
"
\n",
" \n",
" | 4695 | \n",
" 4695 | \n",
" Strikeout | \n",
"
\n",
" \n",
" | 4696 | \n",
" 4696 | \n",
" Arjuna | \n",
"
\n",
" \n",
" | 4697 | \n",
" 4697 | \n",
" Scarlet Weather Rhapsody | \n",
"
\n",
" \n",
" | 4698 | \n",
" 4698 | \n",
" Voetbal International | \n",
"
\n",
" \n",
" | 4699 | \n",
" 4699 | \n",
" Death Row Records | \n",
"
\n",
" \n",
" | 4700 | \n",
" 4700 | \n",
" Shashlik | \n",
"
\n",
" \n",
" | 4701 | \n",
" 4701 | \n",
" Scarecrow | \n",
"
\n",
" \n",
" | 4702 | \n",
" 4702 | \n",
" Akame ga Kill! | \n",
"
\n",
" \n",
" | 4703 | \n",
" 4703 | \n",
" DJ Hero 2 | \n",
"
\n",
" \n",
" | 4704 | \n",
" 4704 | \n",
" Gong | \n",
"
\n",
" \n",
" | 4705 | \n",
" 4705 | \n",
" SCP – Containment Breach | \n",
"
\n",
" \n",
" | 4706 | \n",
" 4706 | \n",
" Friday the 13th: Day of Death | \n",
"
\n",
" \n",
" | 4707 | \n",
" 4707 | \n",
" Poncho | \n",
"
\n",
" \n",
" | 4708 | \n",
" 4708 | \n",
" Visual novel | \n",
"
\n",
" \n",
" | 4709 | \n",
" 4709 | \n",
" Luge | \n",
"
\n",
" \n",
" | 4710 | \n",
" 4710 | \n",
" Mammal | \n",
"
\n",
" \n",
" | 4711 | \n",
" 4711 | \n",
" Aston Martin V8 Vantage (2005) | \n",
"
\n",
" \n",
" | 4712 | \n",
" 4712 | \n",
" Russian pyramid | \n",
"
\n",
" \n",
" | 4713 | \n",
" 4713 | \n",
" PBS Kids | \n",
"
\n",
" \n",
" | 4714 | \n",
" 4714 | \n",
" Air Gear | \n",
"
\n",
" \n",
" | 4715 | \n",
" 4715 | \n",
" Pina Records | \n",
"
\n",
" \n",
"
\n",
"
4716 rows × 2 columns
\n",
"
"
],
"text/plain": [
" label_id label_name\n",
"0 0 Games\n",
"1 1 Vehicle\n",
"2 2 Video game\n",
"3 3 Concert\n",
"4 4 Car\n",
"5 5 Dance\n",
"6 6 Animation\n",
"7 7 Musician\n",
"8 8 Football\n",
"9 9 Music video\n",
"10 10 Animal\n",
"11 11 Motorsport\n",
"12 12 Food\n",
"13 13 Musical ensemble\n",
"14 14 Guitar\n",
"15 15 Cartoon\n",
"16 16 Performance art\n",
"17 17 Racing\n",
"18 18 Outdoor recreation\n",
"19 19 PC game\n",
"20 20 Trailer\n",
"21 21 Stadium\n",
"22 22 Nature\n",
"23 23 Mobile phone\n",
"24 24 String instrument\n",
"25 25 Toy\n",
"26 26 Cooking\n",
"27 27 Motorcycle\n",
"28 28 Fashion\n",
"29 29 Smartphone\n",
"... ... ...\n",
"4686 4686 Nisekoi\n",
"4687 4687 Hollywood Palladium\n",
"4688 4688 Skullgirls\n",
"4689 4689 Al-Masjid an-Nabawi\n",
"4690 4690 Shadow Fighter\n",
"4691 4691 Reptile\n",
"4692 4692 Abercrombie & Fitch\n",
"4693 4693 Daffy Duck\n",
"4694 4694 Cruelty to animals\n",
"4695 4695 Strikeout\n",
"4696 4696 Arjuna\n",
"4697 4697 Scarlet Weather Rhapsody\n",
"4698 4698 Voetbal International\n",
"4699 4699 Death Row Records\n",
"4700 4700 Shashlik\n",
"4701 4701 Scarecrow\n",
"4702 4702 Akame ga Kill!\n",
"4703 4703 DJ Hero 2\n",
"4704 4704 Gong\n",
"4705 4705 SCP – Containment Breach\n",
"4706 4706 Friday the 13th: Day of Death\n",
"4707 4707 Poncho\n",
"4708 4708 Visual novel\n",
"4709 4709 Luge\n",
"4710 4710 Mammal\n",
"4711 4711 Aston Martin V8 Vantage (2005)\n",
"4712 4712 Russian pyramid\n",
"4713 4713 PBS Kids\n",
"4714 4714 Air Gear\n",
"4715 4715 Pina Records\n",
"\n",
"[4716 rows x 2 columns]"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"label_names"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"label = {}\n",
"for item in label_names.iterrows():\n",
" label_id = item[1].label_id\n",
" label_name = item[1].label_name\n",
" label[label_id] = label_name"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Label details"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/html": [
"\n",
"
\n",
" \n",
" \n",
" | \n",
" Index | \n",
" TrainVideoCount | \n",
" KnowledgeGraphId | \n",
" Name | \n",
" WikiUrl | \n",
" Vertical1 | \n",
" Vertical2 | \n",
" Vertical3 | \n",
" WikiDescription | \n",
"
\n",
" \n",
" \n",
" \n",
" | 0 | \n",
" 0 | \n",
" 865174 | \n",
" /m/03bt1gh | \n",
" Game | \n",
" https://en.wikipedia.org/wiki/Game | \n",
" Games | \n",
" NaN | \n",
" NaN | \n",
" A game is structured form of play, usually und... | \n",
"
\n",
" \n",
" | 1 | \n",
" 1 | \n",
" 683166 | \n",
" /m/07yv9 | \n",
" Vehicle | \n",
" https://en.wikipedia.org/wiki/Vehicle | \n",
" Autos & Vehicles | \n",
" NaN | \n",
" NaN | \n",
" A vehicle is a mobile machine that transports ... | \n",
"
\n",
" \n",
" | 2 | \n",
" 2 | \n",
" 522427 | \n",
" /m/01mw1 | \n",
" Video game | \n",
" https://en.wikipedia.org/wiki/Video_game | \n",
" Games | \n",
" NaN | \n",
" NaN | \n",
" A video game is an electronic game that involv... | \n",
"
\n",
" \n",
" | 3 | \n",
" 3 | \n",
" 497487 | \n",
" /m/01jddz | \n",
" Concert | \n",
" https://en.wikipedia.org/wiki/Concert | \n",
" Arts & Entertainment | \n",
" NaN | \n",
" NaN | \n",
" A concert is a live music performance in front... | \n",
"
\n",
" \n",
" | 4 | \n",
" 4 | \n",
" 373952 | \n",
" /m/0k4j | \n",
" Car | \n",
" https://en.wikipedia.org/wiki/Car | \n",
" Autos & Vehicles | \n",
" NaN | \n",
" NaN | \n",
" A car is a wheeled, self-powered motor vehicle... | \n",
"
\n",
" \n",
" | 5 | \n",
" 5 | \n",
" 305143 | \n",
" /m/026bk | \n",
" Dance | \n",
" https://en.wikipedia.org/wiki/Dance | \n",
" Arts & Entertainment | \n",
" NaN | \n",
" NaN | \n",
" Dance is a performance art form consisting of ... | \n",
"
\n",
" \n",
" | 6 | \n",
" 6 | \n",
" 299054 | \n",
" /m/0hcr | \n",
" Animation | \n",
" https://en.wikipedia.org/wiki/Animation | \n",
" Arts & Entertainment | \n",
" NaN | \n",
" NaN | \n",
" Animation is the process of making the illusio... | \n",
"
\n",
" \n",
" | 7 | \n",
" 7 | \n",
" 296577 | \n",
" /m/09jwl | \n",
" Musician | \n",
" https://en.wikipedia.org/wiki/Musician | \n",
" Arts & Entertainment | \n",
" NaN | \n",
" NaN | \n",
" A musician is a person who plays a musical ins... | \n",
"
\n",
" \n",
" | 8 | \n",
" 8 | \n",
" 219506 | \n",
" /m/02vx4 | \n",
" Association football | \n",
" https://en.wikipedia.org/wiki/Association_foot... | \n",
" Sports | \n",
" NaN | \n",
" NaN | \n",
" Association football, more commonly known as f... | \n",
"
\n",
" \n",
" | 9 | \n",
" 9 | \n",
" 217037 | \n",
" /m/0mdxd | \n",
" Music video | \n",
" https://en.wikipedia.org/wiki/Music_video | \n",
" Arts & Entertainment | \n",
" NaN | \n",
" NaN | \n",
" A music video is a short film integrating a so... | \n",
"
\n",
" \n",
" | 10 | \n",
" 10 | \n",
" 203700 | \n",
" /m/0jbk | \n",
" Animal | \n",
" https://en.wikipedia.org/wiki/Animal | \n",
" Pets & Animals | \n",
" NaN | \n",
" NaN | \n",
" Animals are multicellular, eukaryotic organism... | \n",
"
\n",
" \n",
" | 11 | \n",
" 11 | \n",
" 198630 | \n",
" /m/0410tth | \n",
" Motorsport | \n",
" https://en.wikipedia.org/wiki/Motorsport | \n",
" Sports | \n",
" NaN | \n",
" NaN | \n",
" Motorsport or motorsports is a global term use... | \n",
"
\n",
" \n",
" | 12 | \n",
" 12 | \n",
" 196108 | \n",
" /m/02wbm | \n",
" Food | \n",
" https://en.wikipedia.org/wiki/Food | \n",
" Food & Drink | \n",
" NaN | \n",
" NaN | \n",
" Food is any substance consumed to provide nutr... | \n",
"
\n",
" \n",
" | 13 | \n",
" 13 | \n",
" 170351 | \n",
" /m/05229 | \n",
" Musical ensemble | \n",
" https://en.wikipedia.org/wiki/Musical_ensemble | \n",
" Arts & Entertainment | \n",
" NaN | \n",
" NaN | \n",
" A musical ensemble, also known as a music grou... | \n",
"
\n",
" \n",
" | 14 | \n",
" 14 | \n",
" 162864 | \n",
" /m/0342h | \n",
" Guitar | \n",
" https://en.wikipedia.org/wiki/Guitar | \n",
" Arts & Entertainment | \n",
" NaN | \n",
" NaN | \n",
" The guitar is a musical instrument classified ... | \n",
"
\n",
" \n",
" | 15 | \n",
" 15 | \n",
" 162157 | \n",
" /m/0215n | \n",
" Cartoon | \n",
" https://en.wikipedia.org/wiki/Cartoon | \n",
" Arts & Entertainment | \n",
" NaN | \n",
" NaN | \n",
" A cartoon is a type of two-dimensional illustr... | \n",
"
\n",
" \n",
" | 16 | \n",
" 16 | \n",
" 160383 | \n",
" /m/01350r | \n",
" Performance art | \n",
" https://en.wikipedia.org/wiki/Performance_art | \n",
" Arts & Entertainment | \n",
" NaN | \n",
" NaN | \n",
" Performance art is a performance presented to ... | \n",
"
\n",
" \n",
" | 17 | \n",
" 17 | \n",
" 121318 | \n",
" /m/0dfbw | \n",
" Racing | \n",
" https://en.wikipedia.org/wiki/Racing | \n",
" Sports | \n",
" NaN | \n",
" NaN | \n",
" In sport, racing is a competition of speed, ag... | \n",
"
\n",
" \n",
" | 18 | \n",
" 18 | \n",
" 110787 | \n",
" /m/05b0n7k | \n",
" Outdoor recreation | \n",
" https://en.wikipedia.org/wiki/Outdoor_recreation | \n",
" Hobbies & Leisure | \n",
" NaN | \n",
" NaN | \n",
" Outdoor recreation or outdoor activity refers ... | \n",
"
\n",
" \n",
" | 19 | \n",
" 19 | \n",
" 107038 | \n",
" /m/04tr8s | \n",
" PC game | \n",
" https://en.wikipedia.org/wiki/PC_game | \n",
" Games | \n",
" NaN | \n",
" NaN | \n",
" PC games, also known as computer games or pers... | \n",
"
\n",
" \n",
" | 20 | \n",
" 20 | \n",
" 97912 | \n",
" /m/03hdbf | \n",
" Trailer (promotion) | \n",
" https://en.wikipedia.org/wiki/Trailer_(promotion) | \n",
" Arts & Entertainment | \n",
" NaN | \n",
" NaN | \n",
" A trailer is an advertisement or a commercial ... | \n",
"
\n",
" \n",
" | 21 | \n",
" 21 | \n",
" 94966 | \n",
" /m/019cfy | \n",
" Stadium | \n",
" https://en.wikipedia.org/wiki/Stadium | \n",
" Arts & Entertainment | \n",
" NaN | \n",
" NaN | \n",
" A stadium is a place or venue for outdoor spor... | \n",
"
\n",
" \n",
" | 22 | \n",
" 22 | \n",
" 92866 | \n",
" /m/05h0n | \n",
" Nature | \n",
" https://en.wikipedia.org/wiki/Nature | \n",
" Science | \n",
" NaN | \n",
" NaN | \n",
" Nature, in the broadest sense, is the natural,... | \n",
"
\n",
" \n",
" | 23 | \n",
" 23 | \n",
" 90233 | \n",
" /m/050k8 | \n",
" Mobile phone | \n",
" https://en.wikipedia.org/wiki/Mobile_phone | \n",
" Internet & Telecom | \n",
" NaN | \n",
" NaN | \n",
" A mobile phone is a portable telephone that ca... | \n",
"
\n",
" \n",
" | 24 | \n",
" 24 | \n",
" 85888 | \n",
" /m/0d8lm | \n",
" String instrument | \n",
" https://en.wikipedia.org/wiki/String_instrument | \n",
" Arts & Entertainment | \n",
" NaN | \n",
" NaN | \n",
" String instruments, stringed instruments, or c... | \n",
"
\n",
" \n",
" | 25 | \n",
" 25 | \n",
" 84833 | \n",
" /m/0138tl | \n",
" Toy | \n",
" https://en.wikipedia.org/wiki/Toy | \n",
" Shopping | \n",
" NaN | \n",
" NaN | \n",
" A toy is an item that is generally used for ch... | \n",
"
\n",
" \n",
" | 26 | \n",
" 26 | \n",
" 84600 | \n",
" /m/01mtb | \n",
" Cooking | \n",
" https://en.wikipedia.org/wiki/Cooking | \n",
" Food & Drink | \n",
" NaN | \n",
" NaN | \n",
" Cooking or cookery is the art, technology and ... | \n",
"
\n",
" \n",
" | 27 | \n",
" 27 | \n",
" 84366 | \n",
" /m/04_sv | \n",
" Motorcycle | \n",
" https://en.wikipedia.org/wiki/Motorcycle | \n",
" Autos & Vehicles | \n",
" NaN | \n",
" NaN | \n",
" A motorcycle is a two- or three-wheeled motor ... | \n",
"
\n",
" \n",
" | 28 | \n",
" 28 | \n",
" 84355 | \n",
" /m/032tl | \n",
" Fashion | \n",
" https://en.wikipedia.org/wiki/Fashion | \n",
" Beauty & Fitness | \n",
" NaN | \n",
" NaN | \n",
" Fashion is a popular style or practice, especi... | \n",
"
\n",
" \n",
" | 29 | \n",
" 29 | \n",
" 82217 | \n",
" /m/0169zh | \n",
" Smartphone | \n",
" https://en.wikipedia.org/wiki/Smartphone | \n",
" Internet & Telecom | \n",
" NaN | \n",
" NaN | \n",
" A smartphone is a mobile phone with an advance... | \n",
"
\n",
" \n",
" | ... | \n",
" ... | \n",
" ... | \n",
" ... | \n",
" ... | \n",
" ... | \n",
" ... | \n",
" ... | \n",
" ... | \n",
" ... | \n",
"
\n",
" \n",
" | 4686 | \n",
" 4686 | \n",
" 113 | \n",
" /m/0k3hjq7 | \n",
" Nisekoi | \n",
" https://en.wikipedia.org/wiki/Nisekoi | \n",
" (Unknown) | \n",
" NaN | \n",
" NaN | \n",
" Nisekoi, released in English as Nisekoi: False... | \n",
"
\n",
" \n",
" | 4687 | \n",
" 4687 | \n",
" 113 | \n",
" /m/048hm5 | \n",
" Hollywood Palladium | \n",
" https://en.wikipedia.org/wiki/Hollywood_Palladium | \n",
" Arts & Entertainment | \n",
" NaN | \n",
" NaN | \n",
" Hollywood Palladium is a theater located at 62... | \n",
"
\n",
" \n",
" | 4688 | \n",
" 4688 | \n",
" 112 | \n",
" /m/0gtw3vh | \n",
" Skullgirls | \n",
" https://en.wikipedia.org/wiki/Skullgirls | \n",
" Games | \n",
" NaN | \n",
" NaN | \n",
" Skullgirls is a 2D fighting game developed by ... | \n",
"
\n",
" \n",
" | 4689 | \n",
" 4689 | \n",
" 112 | \n",
" /m/024902 | \n",
" Al-Masjid an-Nabawi | \n",
" https://en.wikipedia.org/wiki/Al-Masjid_an-Nabawi | \n",
" Travel | \n",
" People & Society | \n",
" NaN | \n",
" Al-Masjid an-Nabawī is a mosque established an... | \n",
"
\n",
" \n",
" | 4690 | \n",
" 4690 | \n",
" 111 | \n",
" /m/0fv8j0 | \n",
" Shadow Fighter (video game) | \n",
" https://en.wikipedia.org/wiki/Shadow_Fighter_(... | \n",
" Games | \n",
" NaN | \n",
" NaN | \n",
" Shadow Fighter is a computer game for the Comm... | \n",
"
\n",
" \n",
" | 4691 | \n",
" 4691 | \n",
" 111 | \n",
" /m/044p09 | \n",
" Reptile (Mortal Kombat) | \n",
" https://en.wikipedia.org/wiki/Reptile_(Mortal_... | \n",
" Games | \n",
" NaN | \n",
" NaN | \n",
" Reptile is a video game character from the Mor... | \n",
"
\n",
" \n",
" | 4692 | \n",
" 4692 | \n",
" 111 | \n",
" /m/02z2m_ | \n",
" Abercrombie & Fitch | \n",
" https://en.wikipedia.org/wiki/Abercrombie_%26_... | \n",
" Shopping | \n",
" NaN | \n",
" NaN | \n",
" Abercrombie & Fitch is an American retailer th... | \n",
"
\n",
" \n",
" | 4693 | \n",
" 4693 | \n",
" 110 | \n",
" /m/0dng4 | \n",
" Daffy Duck | \n",
" https://en.wikipedia.org/wiki/Daffy_Duck | \n",
" Arts & Entertainment | \n",
" NaN | \n",
" NaN | \n",
" Daffy Duck is an animated cartoon character pr... | \n",
"
\n",
" \n",
" | 4694 | \n",
" 4694 | \n",
" 110 | \n",
" /m/032nch | \n",
" Cruelty to animals | \n",
" https://en.wikipedia.org/wiki/Cruelty_to_animals | \n",
" Pets & Animals | \n",
" NaN | \n",
" NaN | \n",
" Cruelty to animals, also called animal abuse o... | \n",
"
\n",
" \n",
" | 4695 | \n",
" 4695 | \n",
" 110 | \n",
" /m/01k796 | \n",
" Strikeout | \n",
" https://en.wikipedia.org/wiki/Strikeout | \n",
" Sports | \n",
" NaN | \n",
" NaN | \n",
" In baseball or softball, a strikeout occurs wh... | \n",
"
\n",
" \n",
" | 4696 | \n",
" 4696 | \n",
" 109 | \n",
" /m/0pjz3 | \n",
" Arjuna | \n",
" https://en.wikipedia.org/wiki/Arjuna | \n",
" People & Society | \n",
" Reference | \n",
" NaN | \n",
" Arjuna along with Krishna is the protagonist o... | \n",
"
\n",
" \n",
" | 4697 | \n",
" 4697 | \n",
" 108 | \n",
" /m/0661mn1 | \n",
" Scarlet Weather Rhapsody | \n",
" https://en.wikipedia.org/wiki/Scarlet_Weather_... | \n",
" Arts & Entertainment | \n",
" NaN | \n",
" NaN | \n",
" Touhou Hisouten ~ Scarlet Weather Rhapsody. is... | \n",
"
\n",
" \n",
" | 4698 | \n",
" 4698 | \n",
" 108 | \n",
" /m/02z1c71 | \n",
" Voetbal International | \n",
" https://en.wikipedia.org/wiki/Voetbal_Internat... | \n",
" Books & Literature | \n",
" Sports | \n",
" NaN | \n",
" Voetbal International is a Dutch football maga... | \n",
"
\n",
" \n",
" | 4699 | \n",
" 4699 | \n",
" 108 | \n",
" /m/01n2m6 | \n",
" Death Row Records | \n",
" https://en.wikipedia.org/wiki/Death_Row_Records | \n",
" Arts & Entertainment | \n",
" NaN | \n",
" NaN | \n",
" Death Row Records is a record company founded ... | \n",
"
\n",
" \n",
" | 4700 | \n",
" 4700 | \n",
" 108 | \n",
" /m/02p1p9m | \n",
" Shashlik | \n",
" https://en.wikipedia.org/wiki/Shashlik | \n",
" Food & Drink | \n",
" NaN | \n",
" NaN | \n",
" Shashlik or shashlyk, is a dish of skewered an... | \n",
"
\n",
" \n",
" | 4701 | \n",
" 4701 | \n",
" 107 | \n",
" /m/01tcs0 | \n",
" Scarecrow (DC Comics) | \n",
" https://en.wikipedia.org/wiki/Scarecrow_(DC_Co... | \n",
" Arts & Entertainment | \n",
" NaN | \n",
" NaN | \n",
" The Scarecrow is a fictional supervillain appe... | \n",
"
\n",
" \n",
" | 4702 | \n",
" 4702 | \n",
" 107 | \n",
" /m/0_83vt5 | \n",
" Akame ga Kill! | \n",
" https://en.wikipedia.org/wiki/Akame_ga_Kill! | \n",
" (Unknown) | \n",
" NaN | \n",
" NaN | \n",
" Akame ga Kill! is a Japanese shōnen manga seri... | \n",
"
\n",
" \n",
" | 4703 | \n",
" 4703 | \n",
" 107 | \n",
" /m/0c3_50s | \n",
" DJ Hero 2 | \n",
" https://en.wikipedia.org/wiki/DJ_Hero_2 | \n",
" Games | \n",
" NaN | \n",
" NaN | \n",
" DJ Hero 2 is a rhythm video game and a sequel ... | \n",
"
\n",
" \n",
" | 4704 | \n",
" 4704 | \n",
" 106 | \n",
" /m/0mbct | \n",
" Gong | \n",
" https://en.wikipedia.org/wiki/Gong | \n",
" Arts & Entertainment | \n",
" NaN | \n",
" NaN | \n",
" A gong is an African, East and South East Asia... | \n",
"
\n",
" \n",
" | 4705 | \n",
" 4705 | \n",
" 105 | \n",
" /m/0lq1lzn | \n",
" SCP – Containment Breach | \n",
" https://en.wikipedia.org/wiki/SCP_%E2%80%93_Co... | \n",
" (Unknown) | \n",
" NaN | \n",
" NaN | \n",
" SCP – Containment Breach is an indie supernatu... | \n",
"
\n",
" \n",
" | 4706 | \n",
" 4706 | \n",
" 104 | \n",
" /m/0875jr | \n",
" Friday the 13th (franchise) | \n",
" https://en.wikipedia.org/wiki/Friday_the_13th_... | \n",
" Arts & Entertainment | \n",
" NaN | \n",
" NaN | \n",
" Friday the 13th is an American horror franchis... | \n",
"
\n",
" \n",
" | 4707 | \n",
" 4707 | \n",
" 104 | \n",
" /m/03qtgn | \n",
" Poncho | \n",
" https://en.wikipedia.org/wiki/Poncho | \n",
" Shopping | \n",
" NaN | \n",
" NaN | \n",
" A poncho is an outer garment designed to keep ... | \n",
"
\n",
" \n",
" | 4708 | \n",
" 4708 | \n",
" 103 | \n",
" /m/02stb2 | \n",
" Visual novel | \n",
" https://en.wikipedia.org/wiki/Visual_novel | \n",
" Games | \n",
" NaN | \n",
" NaN | \n",
" A visual novel is an interactive game, featuri... | \n",
"
\n",
" \n",
" | 4709 | \n",
" 4709 | \n",
" 103 | \n",
" /m/09f6b | \n",
" Luge | \n",
" https://en.wikipedia.org/wiki/Luge | \n",
" Sports | \n",
" NaN | \n",
" NaN | \n",
" A luge /ˈluːʒ/ is a small one- or two-person s... | \n",
"
\n",
" \n",
" | 4710 | \n",
" 4710 | \n",
" 103 | \n",
" /m/04rky | \n",
" Mammal | \n",
" https://en.wikipedia.org/wiki/Mammal | \n",
" Science | \n",
" NaN | \n",
" NaN | \n",
" Mammals are any vertebrates within the class M... | \n",
"
\n",
" \n",
" | 4711 | \n",
" 4711 | \n",
" 103 | \n",
" /m/06mk92 | \n",
" Aston Martin Vantage (2005) | \n",
" https://en.wikipedia.org/wiki/Aston_Martin_Van... | \n",
" Autos & Vehicles | \n",
" NaN | \n",
" NaN | \n",
" The Aston Martin Vantage is series of hand-bui... | \n",
"
\n",
" \n",
" | 4712 | \n",
" 4712 | \n",
" 102 | \n",
" /m/048ytx | \n",
" Russian pyramid | \n",
" https://en.wikipedia.org/wiki/Russian_pyramid | \n",
" Games | \n",
" NaN | \n",
" NaN | \n",
" Russian pyramid, also known as Russian billiar... | \n",
"
\n",
" \n",
" | 4713 | \n",
" 4713 | \n",
" 101 | \n",
" /m/06_zh7 | \n",
" PBS Kids | \n",
" https://en.wikipedia.org/wiki/PBS_Kids | \n",
" Arts & Entertainment | \n",
" People & Society | \n",
" NaN | \n",
" PBS Kids, stylized as PBS KIDS and formerly PT... | \n",
"
\n",
" \n",
" | 4714 | \n",
" 4714 | \n",
" 101 | \n",
" /m/089w_2 | \n",
" Air Gear | \n",
" https://en.wikipedia.org/wiki/Air_Gear | \n",
" Arts & Entertainment | \n",
" NaN | \n",
" NaN | \n",
" Air Gear is a shōnen manga written and illustr... | \n",
"
\n",
" \n",
" | 4715 | \n",
" 4715 | \n",
" 101 | \n",
" /m/02qg86l | \n",
" Pina Records | \n",
" https://en.wikipedia.org/wiki/Pina_Records | \n",
" Arts & Entertainment | \n",
" NaN | \n",
" NaN | \n",
" Pina Records is a Puerto Rican record label fo... | \n",
"
\n",
" \n",
"
\n",
"
4716 rows × 9 columns
\n",
"
"
],
"text/plain": [
" Index TrainVideoCount KnowledgeGraphId Name \\\n",
"0 0 865174 /m/03bt1gh Game \n",
"1 1 683166 /m/07yv9 Vehicle \n",
"2 2 522427 /m/01mw1 Video game \n",
"3 3 497487 /m/01jddz Concert \n",
"4 4 373952 /m/0k4j Car \n",
"5 5 305143 /m/026bk Dance \n",
"6 6 299054 /m/0hcr Animation \n",
"7 7 296577 /m/09jwl Musician \n",
"8 8 219506 /m/02vx4 Association football \n",
"9 9 217037 /m/0mdxd Music video \n",
"10 10 203700 /m/0jbk Animal \n",
"11 11 198630 /m/0410tth Motorsport \n",
"12 12 196108 /m/02wbm Food \n",
"13 13 170351 /m/05229 Musical ensemble \n",
"14 14 162864 /m/0342h Guitar \n",
"15 15 162157 /m/0215n Cartoon \n",
"16 16 160383 /m/01350r Performance art \n",
"17 17 121318 /m/0dfbw Racing \n",
"18 18 110787 /m/05b0n7k Outdoor recreation \n",
"19 19 107038 /m/04tr8s PC game \n",
"20 20 97912 /m/03hdbf Trailer (promotion) \n",
"21 21 94966 /m/019cfy Stadium \n",
"22 22 92866 /m/05h0n Nature \n",
"23 23 90233 /m/050k8 Mobile phone \n",
"24 24 85888 /m/0d8lm String instrument \n",
"25 25 84833 /m/0138tl Toy \n",
"26 26 84600 /m/01mtb Cooking \n",
"27 27 84366 /m/04_sv Motorcycle \n",
"28 28 84355 /m/032tl Fashion \n",
"29 29 82217 /m/0169zh Smartphone \n",
"... ... ... ... ... \n",
"4686 4686 113 /m/0k3hjq7 Nisekoi \n",
"4687 4687 113 /m/048hm5 Hollywood Palladium \n",
"4688 4688 112 /m/0gtw3vh Skullgirls \n",
"4689 4689 112 /m/024902 Al-Masjid an-Nabawi \n",
"4690 4690 111 /m/0fv8j0 Shadow Fighter (video game) \n",
"4691 4691 111 /m/044p09 Reptile (Mortal Kombat) \n",
"4692 4692 111 /m/02z2m_ Abercrombie & Fitch \n",
"4693 4693 110 /m/0dng4 Daffy Duck \n",
"4694 4694 110 /m/032nch Cruelty to animals \n",
"4695 4695 110 /m/01k796 Strikeout \n",
"4696 4696 109 /m/0pjz3 Arjuna \n",
"4697 4697 108 /m/0661mn1 Scarlet Weather Rhapsody \n",
"4698 4698 108 /m/02z1c71 Voetbal International \n",
"4699 4699 108 /m/01n2m6 Death Row Records \n",
"4700 4700 108 /m/02p1p9m Shashlik \n",
"4701 4701 107 /m/01tcs0 Scarecrow (DC Comics) \n",
"4702 4702 107 /m/0_83vt5 Akame ga Kill! \n",
"4703 4703 107 /m/0c3_50s DJ Hero 2 \n",
"4704 4704 106 /m/0mbct Gong \n",
"4705 4705 105 /m/0lq1lzn SCP – Containment Breach \n",
"4706 4706 104 /m/0875jr Friday the 13th (franchise) \n",
"4707 4707 104 /m/03qtgn Poncho \n",
"4708 4708 103 /m/02stb2 Visual novel \n",
"4709 4709 103 /m/09f6b Luge \n",
"4710 4710 103 /m/04rky Mammal \n",
"4711 4711 103 /m/06mk92 Aston Martin Vantage (2005) \n",
"4712 4712 102 /m/048ytx Russian pyramid \n",
"4713 4713 101 /m/06_zh7 PBS Kids \n",
"4714 4714 101 /m/089w_2 Air Gear \n",
"4715 4715 101 /m/02qg86l Pina Records \n",
"\n",
" WikiUrl Vertical1 \\\n",
"0 https://en.wikipedia.org/wiki/Game Games \n",
"1 https://en.wikipedia.org/wiki/Vehicle Autos & Vehicles \n",
"2 https://en.wikipedia.org/wiki/Video_game Games \n",
"3 https://en.wikipedia.org/wiki/Concert Arts & Entertainment \n",
"4 https://en.wikipedia.org/wiki/Car Autos & Vehicles \n",
"5 https://en.wikipedia.org/wiki/Dance Arts & Entertainment \n",
"6 https://en.wikipedia.org/wiki/Animation Arts & Entertainment \n",
"7 https://en.wikipedia.org/wiki/Musician Arts & Entertainment \n",
"8 https://en.wikipedia.org/wiki/Association_foot... Sports \n",
"9 https://en.wikipedia.org/wiki/Music_video Arts & Entertainment \n",
"10 https://en.wikipedia.org/wiki/Animal Pets & Animals \n",
"11 https://en.wikipedia.org/wiki/Motorsport Sports \n",
"12 https://en.wikipedia.org/wiki/Food Food & Drink \n",
"13 https://en.wikipedia.org/wiki/Musical_ensemble Arts & Entertainment \n",
"14 https://en.wikipedia.org/wiki/Guitar Arts & Entertainment \n",
"15 https://en.wikipedia.org/wiki/Cartoon Arts & Entertainment \n",
"16 https://en.wikipedia.org/wiki/Performance_art Arts & Entertainment \n",
"17 https://en.wikipedia.org/wiki/Racing Sports \n",
"18 https://en.wikipedia.org/wiki/Outdoor_recreation Hobbies & Leisure \n",
"19 https://en.wikipedia.org/wiki/PC_game Games \n",
"20 https://en.wikipedia.org/wiki/Trailer_(promotion) Arts & Entertainment \n",
"21 https://en.wikipedia.org/wiki/Stadium Arts & Entertainment \n",
"22 https://en.wikipedia.org/wiki/Nature Science \n",
"23 https://en.wikipedia.org/wiki/Mobile_phone Internet & Telecom \n",
"24 https://en.wikipedia.org/wiki/String_instrument Arts & Entertainment \n",
"25 https://en.wikipedia.org/wiki/Toy Shopping \n",
"26 https://en.wikipedia.org/wiki/Cooking Food & Drink \n",
"27 https://en.wikipedia.org/wiki/Motorcycle Autos & Vehicles \n",
"28 https://en.wikipedia.org/wiki/Fashion Beauty & Fitness \n",
"29 https://en.wikipedia.org/wiki/Smartphone Internet & Telecom \n",
"... ... ... \n",
"4686 https://en.wikipedia.org/wiki/Nisekoi (Unknown) \n",
"4687 https://en.wikipedia.org/wiki/Hollywood_Palladium Arts & Entertainment \n",
"4688 https://en.wikipedia.org/wiki/Skullgirls Games \n",
"4689 https://en.wikipedia.org/wiki/Al-Masjid_an-Nabawi Travel \n",
"4690 https://en.wikipedia.org/wiki/Shadow_Fighter_(... Games \n",
"4691 https://en.wikipedia.org/wiki/Reptile_(Mortal_... Games \n",
"4692 https://en.wikipedia.org/wiki/Abercrombie_%26_... Shopping \n",
"4693 https://en.wikipedia.org/wiki/Daffy_Duck Arts & Entertainment \n",
"4694 https://en.wikipedia.org/wiki/Cruelty_to_animals Pets & Animals \n",
"4695 https://en.wikipedia.org/wiki/Strikeout Sports \n",
"4696 https://en.wikipedia.org/wiki/Arjuna People & Society \n",
"4697 https://en.wikipedia.org/wiki/Scarlet_Weather_... Arts & Entertainment \n",
"4698 https://en.wikipedia.org/wiki/Voetbal_Internat... Books & Literature \n",
"4699 https://en.wikipedia.org/wiki/Death_Row_Records Arts & Entertainment \n",
"4700 https://en.wikipedia.org/wiki/Shashlik Food & Drink \n",
"4701 https://en.wikipedia.org/wiki/Scarecrow_(DC_Co... Arts & Entertainment \n",
"4702 https://en.wikipedia.org/wiki/Akame_ga_Kill! (Unknown) \n",
"4703 https://en.wikipedia.org/wiki/DJ_Hero_2 Games \n",
"4704 https://en.wikipedia.org/wiki/Gong Arts & Entertainment \n",
"4705 https://en.wikipedia.org/wiki/SCP_%E2%80%93_Co... (Unknown) \n",
"4706 https://en.wikipedia.org/wiki/Friday_the_13th_... Arts & Entertainment \n",
"4707 https://en.wikipedia.org/wiki/Poncho Shopping \n",
"4708 https://en.wikipedia.org/wiki/Visual_novel Games \n",
"4709 https://en.wikipedia.org/wiki/Luge Sports \n",
"4710 https://en.wikipedia.org/wiki/Mammal Science \n",
"4711 https://en.wikipedia.org/wiki/Aston_Martin_Van... Autos & Vehicles \n",
"4712 https://en.wikipedia.org/wiki/Russian_pyramid Games \n",
"4713 https://en.wikipedia.org/wiki/PBS_Kids Arts & Entertainment \n",
"4714 https://en.wikipedia.org/wiki/Air_Gear Arts & Entertainment \n",
"4715 https://en.wikipedia.org/wiki/Pina_Records Arts & Entertainment \n",
"\n",
" Vertical2 Vertical3 \\\n",
"0 NaN NaN \n",
"1 NaN NaN \n",
"2 NaN NaN \n",
"3 NaN NaN \n",
"4 NaN NaN \n",
"5 NaN NaN \n",
"6 NaN NaN \n",
"7 NaN NaN \n",
"8 NaN NaN \n",
"9 NaN NaN \n",
"10 NaN NaN \n",
"11 NaN NaN \n",
"12 NaN NaN \n",
"13 NaN NaN \n",
"14 NaN NaN \n",
"15 NaN NaN \n",
"16 NaN NaN \n",
"17 NaN NaN \n",
"18 NaN NaN \n",
"19 NaN NaN \n",
"20 NaN NaN \n",
"21 NaN NaN \n",
"22 NaN NaN \n",
"23 NaN NaN \n",
"24 NaN NaN \n",
"25 NaN NaN \n",
"26 NaN NaN \n",
"27 NaN NaN \n",
"28 NaN NaN \n",
"29 NaN NaN \n",
"... ... ... \n",
"4686 NaN NaN \n",
"4687 NaN NaN \n",
"4688 NaN NaN \n",
"4689 People & Society NaN \n",
"4690 NaN NaN \n",
"4691 NaN NaN \n",
"4692 NaN NaN \n",
"4693 NaN NaN \n",
"4694 NaN NaN \n",
"4695 NaN NaN \n",
"4696 Reference NaN \n",
"4697 NaN NaN \n",
"4698 Sports NaN \n",
"4699 NaN NaN \n",
"4700 NaN NaN \n",
"4701 NaN NaN \n",
"4702 NaN NaN \n",
"4703 NaN NaN \n",
"4704 NaN NaN \n",
"4705 NaN NaN \n",
"4706 NaN NaN \n",
"4707 NaN NaN \n",
"4708 NaN NaN \n",
"4709 NaN NaN \n",
"4710 NaN NaN \n",
"4711 NaN NaN \n",
"4712 NaN NaN \n",
"4713 People & Society NaN \n",
"4714 NaN NaN \n",
"4715 NaN NaN \n",
"\n",
" WikiDescription \n",
"0 A game is structured form of play, usually und... \n",
"1 A vehicle is a mobile machine that transports ... \n",
"2 A video game is an electronic game that involv... \n",
"3 A concert is a live music performance in front... \n",
"4 A car is a wheeled, self-powered motor vehicle... \n",
"5 Dance is a performance art form consisting of ... \n",
"6 Animation is the process of making the illusio... \n",
"7 A musician is a person who plays a musical ins... \n",
"8 Association football, more commonly known as f... \n",
"9 A music video is a short film integrating a so... \n",
"10 Animals are multicellular, eukaryotic organism... \n",
"11 Motorsport or motorsports is a global term use... \n",
"12 Food is any substance consumed to provide nutr... \n",
"13 A musical ensemble, also known as a music grou... \n",
"14 The guitar is a musical instrument classified ... \n",
"15 A cartoon is a type of two-dimensional illustr... \n",
"16 Performance art is a performance presented to ... \n",
"17 In sport, racing is a competition of speed, ag... \n",
"18 Outdoor recreation or outdoor activity refers ... \n",
"19 PC games, also known as computer games or pers... \n",
"20 A trailer is an advertisement or a commercial ... \n",
"21 A stadium is a place or venue for outdoor spor... \n",
"22 Nature, in the broadest sense, is the natural,... \n",
"23 A mobile phone is a portable telephone that ca... \n",
"24 String instruments, stringed instruments, or c... \n",
"25 A toy is an item that is generally used for ch... \n",
"26 Cooking or cookery is the art, technology and ... \n",
"27 A motorcycle is a two- or three-wheeled motor ... \n",
"28 Fashion is a popular style or practice, especi... \n",
"29 A smartphone is a mobile phone with an advance... \n",
"... ... \n",
"4686 Nisekoi, released in English as Nisekoi: False... \n",
"4687 Hollywood Palladium is a theater located at 62... \n",
"4688 Skullgirls is a 2D fighting game developed by ... \n",
"4689 Al-Masjid an-Nabawī is a mosque established an... \n",
"4690 Shadow Fighter is a computer game for the Comm... \n",
"4691 Reptile is a video game character from the Mor... \n",
"4692 Abercrombie & Fitch is an American retailer th... \n",
"4693 Daffy Duck is an animated cartoon character pr... \n",
"4694 Cruelty to animals, also called animal abuse o... \n",
"4695 In baseball or softball, a strikeout occurs wh... \n",
"4696 Arjuna along with Krishna is the protagonist o... \n",
"4697 Touhou Hisouten ~ Scarlet Weather Rhapsody. is... \n",
"4698 Voetbal International is a Dutch football maga... \n",
"4699 Death Row Records is a record company founded ... \n",
"4700 Shashlik or shashlyk, is a dish of skewered an... \n",
"4701 The Scarecrow is a fictional supervillain appe... \n",
"4702 Akame ga Kill! is a Japanese shōnen manga seri... \n",
"4703 DJ Hero 2 is a rhythm video game and a sequel ... \n",
"4704 A gong is an African, East and South East Asia... \n",
"4705 SCP – Containment Breach is an indie supernatu... \n",
"4706 Friday the 13th is an American horror franchis... \n",
"4707 A poncho is an outer garment designed to keep ... \n",
"4708 A visual novel is an interactive game, featuri... \n",
"4709 A luge /ˈluːʒ/ is a small one- or two-person s... \n",
"4710 Mammals are any vertebrates within the class M... \n",
"4711 The Aston Martin Vantage is series of hand-bui... \n",
"4712 Russian pyramid, also known as Russian billiar... \n",
"4713 PBS Kids, stylized as PBS KIDS and formerly PT... \n",
"4714 Air Gear is a shōnen manga written and illustr... \n",
"4715 Pina Records is a Puerto Rican record label fo... \n",
"\n",
"[4716 rows x 9 columns]"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"vocabulary"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"vertical_dict = {}\n",
"with open(\"vertical.tsv\", \"w\") as F:\n",
" lines = []\n",
" for num, item in vocabulary.iterrows():\n",
" index = item[\"Index\"]\n",
" vertical = item[\"Vertical1\"]\n",
" if vertical not in vertical_dict:\n",
" vertical_dict[vertical] = len(vertical_dict)\n",
" lines.append(\"%d\\t%d\\n\" % (index, vertical_dict[vertical]))\n",
" F.writelines(lines)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"25\n"
]
}
],
"source": [
"print len(vertical_dict)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Training labels"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/html": [
"\n",
"
\n",
" \n",
" \n",
" | \n",
" id | \n",
" labels | \n",
"
\n",
" \n",
" \n",
" \n",
" | 0 | \n",
" --DwgB78t-c | \n",
" 16 5 430 | \n",
"
\n",
" \n",
" | 1 | \n",
" --NZRkXBV7k | \n",
" 128 3 39 7 44 13 16 30 | \n",
"
\n",
" \n",
" | 2 | \n",
" --hoQ2sGG4M | \n",
" 694 | \n",
"
\n",
" \n",
" | 3 | \n",
" --ABhs9ik7c | \n",
" 2232 1 4 517 270 | \n",
"
\n",
" \n",
" | 4 | \n",
" --sBoaqBlzA | \n",
" 45 125 | \n",
"
\n",
" \n",
" | 5 | \n",
" --7h1S4neDM | \n",
" 0 48 10 356 | \n",
"
\n",
" \n",
" | 6 | \n",
" --F672jfCMo | \n",
" 488 1067 1078 | \n",
"
\n",
" \n",
" | 7 | \n",
" --ezS5q-mZg | \n",
" 1 962 12 625 | \n",
"
\n",
" \n",
" | 8 | \n",
" --XT8O4T3Wc | \n",
" 379 6 | \n",
"
\n",
" \n",
" | 9 | \n",
" --Qgwg7mGZY | \n",
" 0 33 2 2379 3423 | \n",
"
\n",
" \n",
" | 10 | \n",
" --_I8vffnIw | \n",
" 768 37 72 330 331 29 373 23 189 | \n",
"
\n",
" \n",
" | 11 | \n",
" --VDzNzHHic | \n",
" 20 | \n",
"
\n",
" \n",
" | 12 | \n",
" --pQCGgGjE8 | \n",
" 0 2 836 432 665 2779 | \n",
"
\n",
" \n",
" | 13 | \n",
" --amuSGpp94 | \n",
" 118 | \n",
"
\n",
" \n",
" | 14 | \n",
" --mkDBHaFIs | \n",
" 6 | \n",
"
\n",
" \n",
" | 15 | \n",
" --DyrrdZbUk | \n",
" 1 114 1395 4 3409 | \n",
"
\n",
" \n",
" | 16 | \n",
" --cPdta0k4Y | \n",
" 276 | \n",
"
\n",
" \n",
" | 17 | \n",
" --5iMsHNuDE | \n",
" 1206 6 967 | \n",
"
\n",
" \n",
" | 18 | \n",
" --f5fEIOW1Q | \n",
" 9 | \n",
"
\n",
" \n",
" | 19 | \n",
" --jcB8BTawk | \n",
" 5 31 | \n",
"
\n",
" \n",
" | 20 | \n",
" --0soWyo5gQ | \n",
" 1 403 4 1511 975 | \n",
"
\n",
" \n",
" | 21 | \n",
" --8MxffdH9k | \n",
" 1 4 | \n",
"
\n",
" \n",
" | 22 | \n",
" --j9Z0JKrjE | \n",
" 193 436 45 173 | \n",
"
\n",
" \n",
" | 23 | \n",
" --o719vqu6Q | \n",
" 0 49 168 60 54 | \n",
"
\n",
" \n",
" | 24 | \n",
" --p02WMptcA | \n",
" 49 | \n",
"
\n",
" \n",
" | 25 | \n",
" --0pQywa6OM | \n",
" 1139 | \n",
"
\n",
" \n",
" | 26 | \n",
" --sW2Fk-LmY | \n",
" 0 2 35 | \n",
"
\n",
" \n",
" | 27 | \n",
" --iEJ1NTTHQ | \n",
" 8 813 79 | \n",
"
\n",
" \n",
" | 28 | \n",
" --39-Ml5Ylw | \n",
" 99 3581 | \n",
"
\n",
" \n",
" | 29 | \n",
" --udooZCtNE | \n",
" 200 25 | \n",
"
\n",
" \n",
" | ... | \n",
" ... | \n",
" ... | \n",
"
\n",
" \n",
" | 4906630 | \n",
" zzCGK4T43sI | \n",
" 16 3 | \n",
"
\n",
" \n",
" | 4906631 | \n",
" zz9AvyGAFhg | \n",
" 0 2 35 | \n",
"
\n",
" \n",
" | 4906632 | \n",
" zzAhVFy3o5s | \n",
" 233 222 | \n",
"
\n",
" \n",
" | 4906633 | \n",
" zzhKCCjBtN0 | \n",
" 208 170 830 246 | \n",
"
\n",
" \n",
" | 4906634 | \n",
" zzQfq7MBrfk | \n",
" 80 10 438 247 | \n",
"
\n",
" \n",
" | 4906635 | \n",
" zz5H1l4fTRs | \n",
" 0 33 2 1382 | \n",
"
\n",
" \n",
" | 4906636 | \n",
" zztyezN9m80 | \n",
" 1002 | \n",
"
\n",
" \n",
" | 4906637 | \n",
" zz9urhmAd6M | \n",
" 16 3 5 | \n",
"
\n",
" \n",
" | 4906638 | \n",
" zzieB3E90oc | \n",
" 49 3 13 38 7 | \n",
"
\n",
" \n",
" | 4906639 | \n",
" zz-AlFFQAbI | \n",
" 354 713 12 2482 466 1047 | \n",
"
\n",
" \n",
" | 4906640 | \n",
" zzYbFq5qF8w | \n",
" 400 2017 972 | \n",
"
\n",
" \n",
" | 4906641 | \n",
" zzyTPLaoWxo | \n",
" 3 38 39 7 13 49 | \n",
"
\n",
" \n",
" | 4906642 | \n",
" zzp9Q2bQhck | \n",
" 1 4 202 11 17 164 92 447 | \n",
"
\n",
" \n",
" | 4906643 | \n",
" zz4L8mQ0Zr8 | \n",
" 112 59 316 | \n",
"
\n",
" \n",
" | 4906644 | \n",
" zzxF2j31aBE | \n",
" 1 1323 11 172 17 278 728 57 27 669 | \n",
"
\n",
" \n",
" | 4906645 | \n",
" zzTw-YSIp1g | \n",
" 163 | \n",
"
\n",
" \n",
" | 4906646 | \n",
" zznsbYnqvyU | \n",
" 0 2 34 1069 19 118 | \n",
"
\n",
" \n",
" | 4906647 | \n",
" zz9z1DR2RXA | \n",
" 24 14 63 | \n",
"
\n",
" \n",
" | 4906648 | \n",
" zzk5fGf0nMw | \n",
" 8 | \n",
"
\n",
" \n",
" | 4906649 | \n",
" zzrCAKdNth4 | \n",
" 744 1794 3182 | \n",
"
\n",
" \n",
" | 4906650 | \n",
" zzX28lrE9Fc | \n",
" 46 | \n",
"
\n",
" \n",
" | 4906651 | \n",
" zzMFAaqNGX4 | \n",
" 208 | \n",
"
\n",
" \n",
" | 4906652 | \n",
" zzhc0nTbC4Q | \n",
" 49 5 95 | \n",
"
\n",
" \n",
" | 4906653 | \n",
" zztFMPeS4KU | \n",
" 20 | \n",
"
\n",
" \n",
" | 4906654 | \n",
" zz5R0c6JHBk | \n",
" 56 106 28 | \n",
"
\n",
" \n",
" | 4906655 | \n",
" zzRKzJ5iCRs | \n",
" 416 113 652 339 580 | \n",
"
\n",
" \n",
" | 4906656 | \n",
" zzDg9v6733A | \n",
" 1 4 | \n",
"
\n",
" \n",
" | 4906657 | \n",
" zzZl6-AxXv8 | \n",
" 0 60 | \n",
"
\n",
" \n",
" | 4906658 | \n",
" zzoW1afkGKk | \n",
" 2033 1083 3959 | \n",
"
\n",
" \n",
" | 4906659 | \n",
" zzfubODk_8E | \n",
" 10 6 15 | \n",
"
\n",
" \n",
"
\n",
"
4906660 rows × 2 columns
\n",
"
"
],
"text/plain": [
" id labels\n",
"0 --DwgB78t-c 16 5 430\n",
"1 --NZRkXBV7k 128 3 39 7 44 13 16 30\n",
"2 --hoQ2sGG4M 694\n",
"3 --ABhs9ik7c 2232 1 4 517 270\n",
"4 --sBoaqBlzA 45 125\n",
"5 --7h1S4neDM 0 48 10 356\n",
"6 --F672jfCMo 488 1067 1078\n",
"7 --ezS5q-mZg 1 962 12 625\n",
"8 --XT8O4T3Wc 379 6\n",
"9 --Qgwg7mGZY 0 33 2 2379 3423\n",
"10 --_I8vffnIw 768 37 72 330 331 29 373 23 189\n",
"11 --VDzNzHHic 20\n",
"12 --pQCGgGjE8 0 2 836 432 665 2779\n",
"13 --amuSGpp94 118\n",
"14 --mkDBHaFIs 6\n",
"15 --DyrrdZbUk 1 114 1395 4 3409\n",
"16 --cPdta0k4Y 276\n",
"17 --5iMsHNuDE 1206 6 967\n",
"18 --f5fEIOW1Q 9\n",
"19 --jcB8BTawk 5 31\n",
"20 --0soWyo5gQ 1 403 4 1511 975\n",
"21 --8MxffdH9k 1 4\n",
"22 --j9Z0JKrjE 193 436 45 173\n",
"23 --o719vqu6Q 0 49 168 60 54\n",
"24 --p02WMptcA 49\n",
"25 --0pQywa6OM 1139\n",
"26 --sW2Fk-LmY 0 2 35\n",
"27 --iEJ1NTTHQ 8 813 79\n",
"28 --39-Ml5Ylw 99 3581\n",
"29 --udooZCtNE 200 25\n",
"... ... ...\n",
"4906630 zzCGK4T43sI 16 3\n",
"4906631 zz9AvyGAFhg 0 2 35\n",
"4906632 zzAhVFy3o5s 233 222\n",
"4906633 zzhKCCjBtN0 208 170 830 246\n",
"4906634 zzQfq7MBrfk 80 10 438 247\n",
"4906635 zz5H1l4fTRs 0 33 2 1382\n",
"4906636 zztyezN9m80 1002\n",
"4906637 zz9urhmAd6M 16 3 5\n",
"4906638 zzieB3E90oc 49 3 13 38 7\n",
"4906639 zz-AlFFQAbI 354 713 12 2482 466 1047\n",
"4906640 zzYbFq5qF8w 400 2017 972\n",
"4906641 zzyTPLaoWxo 3 38 39 7 13 49\n",
"4906642 zzp9Q2bQhck 1 4 202 11 17 164 92 447\n",
"4906643 zz4L8mQ0Zr8 112 59 316\n",
"4906644 zzxF2j31aBE 1 1323 11 172 17 278 728 57 27 669\n",
"4906645 zzTw-YSIp1g 163\n",
"4906646 zznsbYnqvyU 0 2 34 1069 19 118\n",
"4906647 zz9z1DR2RXA 24 14 63\n",
"4906648 zzk5fGf0nMw 8\n",
"4906649 zzrCAKdNth4 744 1794 3182\n",
"4906650 zzX28lrE9Fc 46\n",
"4906651 zzMFAaqNGX4 208\n",
"4906652 zzhc0nTbC4Q 49 5 95\n",
"4906653 zztFMPeS4KU 20\n",
"4906654 zz5R0c6JHBk 56 106 28\n",
"4906655 zzRKzJ5iCRs 416 113 652 339 580\n",
"4906656 zzDg9v6733A 1 4\n",
"4906657 zzZl6-AxXv8 0 60\n",
"4906658 zzoW1afkGKk 2033 1083 3959\n",
"4906659 zzfubODk_8E 10 6 15\n",
"\n",
"[4906660 rows x 2 columns]"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"train_labels"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### show some of the label combinations"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"collapsed": false,
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"5-Dance / 16-Performance art / 430-Ballroom dance / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / \n",
"694-Tai chi / \n",
"1-Vehicle / 4-Car / 270-Sedan / 517-Audi / 2232-Audi A3 / \n",
"45-Cosmetics / 125-Television / \n",
"0-Games / 10-Animal / 48-Pet / 356-Star / \n",
"488-Door / 1067-Lock / 1078-Installation art / \n",
"1-Vehicle / 12-Food / 625-Factory / 962-McDonald's / \n",
"6-Animation / 379-Touhou Project / \n",
"0-Games / 2-Video game / 33-Weapon / 2379-Tom Clancy's Ghost Recon / 3423-Tom Clancy's Ghost Recon: Future Soldier / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 189-iPad / 330-Money / 331-iPod touch / 373-Clash of Clans / 768-Diamond / \n",
"20-Trailer / \n",
"0-Games / 2-Video game / 432-Kingdom Hearts / 665-Kingdom Hearts / 836-Sora / 2779-Universe of Kingdom Hearts / \n",
"118-Grand Theft Auto V / \n",
"6-Animation / \n",
"1-Vehicle / 4-Car / 114-Engine / 1395-Valve / 3409-Nissan Altima / \n",
"276-Room / \n",
"6-Animation / 967-Text / 1206-Cinema 4D / \n",
"9-Music video / \n",
"5-Dance / 31-Disc jockey / \n",
"1-Vehicle / 4-Car / 403-Toyota / 975-Dune / 1511-Toyota Land Cruiser / \n",
"1-Vehicle / 4-Car / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 436-Concealer / \n",
"0-Games / 49-School / 54-Highlight film / 60-Basketball / 168-Basketball moves / \n",
"49-School / \n",
"1139-Belle / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"8-Football / 79-American football / 813-Running back / \n",
"99-Christmas / 3581-Rudolph the Red-Nosed Reindeer / \n",
"25-Toy / 200-Doll / \n",
"1427-TalesRunner / \n",
"983-Lexus / \n",
"59-Winter sport / 84-Snow / 177-Skiing / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"3-Concert / 5-Dance / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"3-Concert / 5-Dance / 68-Lighting / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 135-Drifting / 212-Highway / 231-Supercar / 705-Forza Motorsport / 1530-Forza Motorsport 4 / 2261-Nissan Silvia / \n",
"5-Dance / \n",
"12-Food / 26-Cooking / 32-Recipe / 226-Dessert / 232-Cake / 236-Baking / 295-Chocolate / 534-Milk / 759-Icing / 985-Strawberry / 1728-Chocolate cake / \n",
"2-Video game / 20-Trailer / \n",
"42-Fishing / 902-Fishing bait / 1018-Salmon / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 1027-Terraria / 2821-Cactus / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 25-Toy / 57-Race track / 92-Radio-controlled model / 167-Four-wheel drive / 174-Farm / 202-Radio-controlled car / 718-Duck / 926-Traxxas / \n",
"1-Vehicle / 17-Racing / 27-Motorcycle / 114-Engine / 1036-Moped / \n",
"2-Video game / 47-Personal computer / 216-Call of Duty: Modern Warfare 3 / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / 265-Electronic keyboard / 397-Organ / 1265-Kyle Kingson / \n",
"3-Concert / 5-Dance / 68-Lighting / 133-Nightclub / \n",
"42-Fishing / 315-Knife / 510-Fishing lure / 3745-Eel / \n",
"7-Musician / 36-Piano / \n",
"337-Advertising / \n",
"0-Games / 447-Sand / \n",
"6-Animation / 2176-Fist of the North Star / 3349-Kenshiro / \n",
"6-Animation / 15-Cartoon / 911-Studio / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 25-Toy / 92-Radio-controlled model / 135-Drifting / 202-Radio-controlled car / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 70-Driving / 952-Chevrolet Corvette / 2885-Chevrolet Corvette (C6) / \n",
"10-Animal / 91-Fish / 225-Lake / 258-Aquarium / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 124-Tractor / 249-Heavy equipment / 252-Tractor pulling / \n",
"6-Animation / 25-Toy / 269-Comic book / 1049-Collectible card game / \n",
"6-Animation / 15-Cartoon / 723-Pig / \n",
"86-Plant / 136-Gardening / 279-Garden / 363-Soil / \n",
"12-Food / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 70-Driving / 135-Drifting / \n",
"0-Games / 19-PC game / 51-Strategy video game / 165-League of Legends / \n",
"1-Vehicle / 4-Car / 125-Television / 950-Blu-ray disc / \n",
"1-Vehicle / 4-Car / 74-Truck / 158-Tire / 1429-Dump truck / \n",
"28-Fashion / \n",
"253-Carnival / 641-Mask / \n",
"12-Food / \n",
"174-Farm / \n",
"5-Dance / 16-Performance art / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 47-Personal computer / 145-Tablet computer / \n",
"131-Computer / 497-Macintosh / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 711-Gran Turismo / 790-Nissan GT-R / 1519-Gran Turismo 6 / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 231-Supercar / 569-Lamborghini / \n",
"1650-The Idolmaster / 1755-The Idolmaster / \n",
"0-Games / 2-Video game / 55-Video game console / 3217-Neo Geo / \n",
"23-Mobile phone / 29-Smartphone / \n",
"712-Outer space / 992-Rocket / 1508-Spacecraft / 1992-Space Shuttle / \n",
"350-Circus / 788-Juggling / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 102-Building / 129-Aviation / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / 1736-Microsoft Flight Simulator / 2136-Boeing 757 / 2391-American Airlines / \n",
"3-Concert / 16-Performance art / \n",
"125-Television / 1172-VHS / \n",
"0-Games / 211-RuneScape / \n",
"46-Choir / 318-Church / \n",
"36-Piano / 53-Keyboard / \n",
"130-Gymnastics / \n",
"45-Cosmetics / 436-Concealer / \n",
"1-Vehicle / 11-Motorsport / 124-Tractor / 156-Agriculture / 174-Farm / 252-Tractor pulling / 617-Maize / \n",
"6-Animation / 15-Cartoon / 4244-Rozen Maiden / \n",
"0-Games / 2-Video game / 19-PC game / 3235-Monopoly / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / 150-Violin / \n",
"1-Vehicle / 50-Aircraft / 129-Aviation / 151-Landing / 451-Cockpit / \n",
"1-Vehicle / 1409-Missile / 1643-Submarine / 3842-Radio-controlled submarine / \n",
"66-Bollywood / \n",
"10-Animal / 80-Horse / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 96-Soldier / 257-Counter-Strike / \n",
"3-Concert / 7-Musician / 30-Drummer / 44-Drums / \n",
"345-Loudspeaker / 484-Subwoofer / \n",
"772-Bowling / 930-Ten-pin bowling / 1217-Bowling ball / \n",
"45-Cosmetics / 238-Nail / 300-Nail art / 304-Nail polish / 307-Hand / 324-Finger / 338-Manicure / \n",
"0-Games / 2-Video game / 43-Call of Duty / 116-Pokémon / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / 3050-Mewtwo / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 2626-Shih Tzu / \n",
"102-Building / 812-Plastic / 1384-Greenhouse / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 189-iPad / 272-iPod / 331-iPod touch / 508-iPhone 5 / \n",
"10-Animal / 71-Dog / 1784-Goblin / \n",
"2071-Indiana Jones / \n",
"0-Games / 489-Board game / 2267-Geometry Dash / 2833-Go / \n",
"1-Vehicle / 62-Train / 64-Transport / 224-Rapid transit / \n",
"1-Vehicle / 4-Car / 706-Deer / 955-Police officer / \n",
"14-Guitar / 24-String instrument / 401-Cello / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 230-Brass instrument / 545-Quartet (ensemble) / 3185-Bronze / \n",
"261-Tool / 521-Metal / 1033-Drill / 1073-Chainsaw / \n",
"9-Music video / \n",
"6-Animation / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / \n",
"6-Animation / \n",
"28-Fashion / 657-Autumn / \n",
"3-Concert / 7-Musician / 1684-Hillsong Church / \n",
"85-Combat / \n",
"67-Cycling / 69-Bicycle / 469-Bag / 835-Backpack / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / \n",
"6-Animation / 15-Cartoon / 116-Pokémon / 186-Pokémon / \n",
"6-Animation / 15-Cartoon / 25-Toy / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 167-Four-wheel drive / 198-Off-road vehicle / 327-Mud / 360-Jeep / 565-Mud bogging / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 138-Call of Duty: Black Ops II / \n",
"1-Vehicle / 4-Car / 263-Light / \n",
"113-House / 416-Apartment / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 20-Trailer / \n",
"1-Vehicle / 11-Motorsport / \n",
"1-Vehicle / 4-Car / 519-Recreational vehicle / \n",
"1-Vehicle / 25-Toy / 92-Radio-controlled model / 140-LEGO / 202-Radio-controlled car / \n",
"1-Vehicle / 4-Car / 114-Engine / 234-Ford / 473-Ford Mustang / 509-Classic car / \n",
"10-Animal / 208-Wood / 540-Insect / 775-Bee / 784-Honey / 1121-Beehive / 2501-Picture frame / \n",
"0-Games / 211-RuneScape / 1178-Quest / \n",
"5-Dance / 16-Performance art / 115-Ballet / \n",
"7-Musician / 36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"0-Games / 8-Football / 21-Stadium / 73-Ball / \n",
"1-Vehicle / 18-Outdoor recreation / 22-Nature / 67-Cycling / 69-Bicycle / 197-Trail / 220-Mountain bike / 284-Mountain biking / \n",
"8-Football / \n",
"3-Concert / 16-Performance art / \n",
"0-Games / 2-Video game / 111-PlayStation 3 / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 114-Engine / \n",
"8-Football / 21-Stadium / \n",
"9-Music video / \n",
"5-Dance / 46-Choir / \n",
"1-Vehicle / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"0-Games / 1-Vehicle / 2-Video game / 34-Action-adventure game / 118-Grand Theft Auto V / 176-PlayStation 4 / 181-Xbox / 314-Xbox One / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 139-Airport / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 85-Combat / 656-Metal Gear / 2698-Metal Gear Rising: Revengeance / \n",
"20-Trailer / \n",
"0-Games / 6-Animation / 51-Strategy video game / 460-Yu-Gi-Oh! Trading Card Game / \n",
"3-Concert / 16-Performance art / \n",
"9-Music video / \n",
"0-Games / 21-Stadium / 54-Highlight film / 81-Athlete / 562-Pitcher / 1111-Home run / \n",
"6-Animation / \n",
"6-Animation / 15-Cartoon / 89-Comics / 294-Batman / 803-DC Comics / 2082-Catwoman / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / 1909-Analog synthesizer / 2521-Jupiter / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 70-Driving / 231-Supercar / 344-Coupé / 1056-Porsche 911 / 1965-Porsche Carrera / 2208-Porsche 911 GT3 / \n",
"6-Animation / 15-Cartoon / 801-Inuyasha / \n",
"12-Food / 26-Cooking / 32-Recipe / 232-Cake / 236-Baking / 3020-Raspberry / \n",
"3-Concert / 38-Orchestra / 1431-Madison Square Garden / \n",
"1-Vehicle / 50-Aircraft / 129-Aviation / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / 471-Wing / 1316-Airbus A330 / \n",
"28-Fashion / 56-Hair / 106-Hairstyle / 583-Wig / \n",
"0-Games / 2-Video game / 229-Halo / 470-Halo 3 / \n",
"56-Hair / 106-Hairstyle / \n",
"6-Animation / 15-Cartoon / 1866-Katara / \n",
"0-Games / 2-Video game / 51-Strategy video game / 791-Magic: The Gathering / \n",
"6-Animation / 25-Toy / 140-LEGO / 477-Transformers / 990-Optimus Prime / \n",
"2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 189-iPad / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"3674-Barcode / 3780-QR code / \n",
"3-Concert / 5-Dance / 21-Stadium / 342-Windows Media Video / \n",
"8-Football / \n",
"9-Music video / \n",
"0-Games / 1-Vehicle / 2-Video game / 74-Truck / 328-Simulation video game / 1099-Euro Truck Simulator 2 / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 2217-Moto G / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 422-Call of Duty: Ghosts / \n",
"75-Wrestling / 85-Combat / \n",
"0-Games / 2-Video game / 75-Wrestling / 127-Xbox 360 / 777-WWE 2K / 2629-WWE SmackDown vs. Raw 2010 / \n",
"3-Concert / 16-Performance art / \n",
"468-Biology / 1110-Cell / \n",
"3-Concert / 5-Dance / \n",
"3-Concert / 7-Musician / \n",
"5-Dance / 115-Ballet / 980-Solo dance / \n",
"28-Fashion / \n",
"3-Concert / 5-Dance / 16-Performance art / 95-Talent show / \n",
"227-Water / \n",
"0-Games / 2-Video game / 474-Destiny / \n",
"2-Video game / 20-Trailer / 34-Action-adventure game / 1144-Castlevania / \n",
"110-Album / \n",
"23-Mobile phone / 29-Smartphone / 101-Telephone / 932-Microsoft Lumia / 3376-Memory card / \n",
"2627-Little Red Riding Hood / \n",
"6-Animation / 123-Naruto / \n",
"1-Vehicle / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 220-Mountain bike / 284-Mountain biking / 520-Bicycle frame / \n",
"0-Games / 60-Basketball / 81-Athlete / 255-Slam dunk / \n",
"961-Sega Genesis / \n",
"5-Dance / 16-Performance art / 98-Festival / \n",
"0-Games / 2-Video game / 241-Fighting game / 764-The King of Fighters / 2714-Iori Yagami / 3160-Kyo Kusanagi / \n",
"0-Games / 2-Video game / 6-Animation / 241-Fighting game / 486-M.U.G.E.N / \n",
"345-Loudspeaker / \n",
"695-Balloon / \n",
"8-Football / \n",
"18-Outdoor recreation / 42-Fishing / 120-Winter / 262-Ice / 698-Fisherman / 896-Largemouth bass / 2836-Walleye / \n",
"1-Vehicle / 50-Aircraft / 923-Boeing 777 / \n",
"1-Vehicle / 4-Car / 349-Chevrolet / 406-Chevrolet / 509-Classic car / 2446-Used car / \n",
"6-Animation / \n",
"0-Games / 8-Football / 79-American football / \n",
"287-Human swimming / 329-Swimming pool / \n",
"0-Games / 2-Video game / 505-Mega Man / 2602-Mega Man X / \n",
"18-Outdoor recreation / 59-Winter sport / 177-Skiing / 305-Ski / 789-Alpine skiing / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"1-Vehicle / \n",
"9-Music video / \n",
"536-Glass / 640-Bottle / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 105-Motorcycling / 114-Engine / 336-Scooter / 1036-Moped / 1448-Vespa / \n",
"130-Gymnastics / 204-Gym / \n",
"60-Basketball / \n",
"223-Weather / \n",
"3-Concert / \n",
"546-Stitch / 607-Lace / 1057-Embroidery / 1804-Sewing needle / \n",
"0-Games / 2-Video game / 1419-GameSpot / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 1278-Labrador Retriever / \n",
"3-Concert / 7-Musician / \n",
"3-Concert / \n",
"3-Concert / \n",
"6-Animation / 15-Cartoon / 97-Drawing / \n",
"0-Games / 2-Video game / 159-World of Warcraft / 171-Warcraft / 849-Tales / \n",
"0-Games / 8-Football / \n",
"0-Games / 2-Video game / 3296-Transistor / \n",
"8-Football / 76-Ball / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 101-Telephone / 272-iPod / \n",
"1-Vehicle / 4-Car / 1324-Roadster / \n",
"56-Hair / 106-Hairstyle / 340-Afro-textured hair / \n",
"8-Football / \n",
"86-Plant / 136-Gardening / 184-Vegetable / 1209-Bean / \n",
"3-Concert / 16-Performance art / \n",
"3-Concert / \n",
"66-Bollywood / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / \n",
"28-Fashion / 206-Shoe / 333-Nike; Inc. / 368-Sneakers / 1423-Nike Air Max / 1999-Eggplant / \n",
"6-Animation / 15-Cartoon / \n",
"6-Animation / \n",
"102-Building / 574-Elevator / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 24-String instrument / 36-Piano / 38-Orchestra / 150-Violin / 376-Flute / 450-Viola / \n",
"3-Concert / 13-Musical ensemble / 16-Performance art / \n",
"61-Art / 97-Drawing / 118-Grand Theft Auto V / 201-Grand Theft Auto: San Andreas / \n",
"410-Teacher / 776-Kindergarten / \n",
"59-Winter sport / 84-Snow / 177-Skiing / 281-Snowboarding / 305-Ski / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"25-Toy / 89-Comics / 435-Action figure / 2121-Marvel Legends / 3109-Clint Barton / \n",
"337-Advertising / \n",
"6-Animation / 123-Naruto / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 38-Orchestra / 39-Drums / \n",
"1578-Red Dead Redemption / 1891-Red Dead / \n",
"12-Food / 232-Cake / 295-Chocolate / 1102-Cake decorating / 1189-Torte / 2941-Wedding cake / \n",
"6-Animation / 15-Cartoon / \n",
"260-Prayer / \n",
"5-Dance / 49-School / 1019-Christ / \n",
"1-Vehicle / 114-Engine / \n",
"33-Weapon / \n",
"6-Animation / 15-Cartoon / 1130-Armour / 2398-Personal armor / 3185-Bronze / \n",
"18-Outdoor recreation / 22-Nature / \n",
"45-Cosmetics / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"13-Musical ensemble / 21-Stadium / 79-American football / 203-Marching band / 209-University / \n",
"1-Vehicle / 4-Car / 2396-Coyote / \n",
"45-Cosmetics / 402-Jewellery / \n",
"83-Skateboarding / 108-Skateboard / 556-Skateboarding trick / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"6-Animation / 15-Cartoon / 294-Batman / \n",
"1584-Kettlebell / \n",
"0-Games / 8-Football / 65-Kick / 73-Ball / 90-Sports game / 180-FIFA 15 / \n",
"125-Television / 223-Weather / \n",
"45-Cosmetics / 193-Eye liner / 195-Mascara / 196-Eye / \n",
"1058-Volcano / \n",
"6-Animation / 15-Cartoon / 61-Art / 97-Drawing / 379-Touhou Project / 408-Sketch / 2162-Marker pen / \n",
"3-Concert / \n",
"3-Concert / 5-Dance / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / \n",
"3-Concert / 7-Musician / \n",
"1254-Cave / \n",
"3-Concert / 133-Nightclub / \n",
"131-Computer / 141-Microsoft Windows / \n",
"6-Animation / 15-Cartoon / 123-Naruto / 183-Manga / \n",
"10-Animal / 1330-Bull riding / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 172-Motocross / 253-Carnival / \n",
"18-Outdoor recreation / 59-Winter sport / 84-Snow / 120-Winter / 177-Skiing / 281-Snowboarding / 305-Ski / 789-Alpine skiing / \n",
"5-Dance / 31-Disc jockey / \n",
"5-Dance / 49-School / \n",
"36-Piano / 265-Electronic keyboard / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / \n",
"9-Music video / \n",
"5-Dance / \n",
"6-Animation / 269-Comic book / 460-Yu-Gi-Oh! Trading Card Game / 494-Card game / \n",
"774-Galaxy / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 2-Video game / 19-PC game / 192-Battlefield / 374-Battlefield 3 / 464-First-person Shooter / \n",
"1-Vehicle / 17-Racing / 50-Aircraft / \n",
"0-Games / 2-Video game / 6-Animation / 2237-Kantai Collection / \n",
"36-Piano / 46-Choir / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"12-Food / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"33-Weapon / 315-Knife / \n",
"22-Nature / 86-Plant / 136-Gardening / 279-Garden / 1023-Rose / \n",
"36-Piano / \n",
"0-Games / 60-Basketball / \n",
"9-Music video / \n",
"121-Photography / \n",
"5-Dance / 78-Wedding / 3693-Groomsman / \n",
"59-Winter sport / 84-Snow / 120-Winter / 177-Skiing / 281-Snowboarding / 789-Alpine skiing / 1830-Downhill / \n",
"3478-Meme / \n",
"0-Games / 2-Video game / 43-Call of Duty / 194-Call of Duty: Modern Warfare 2 / 2074-Killzone / \n",
"49-School / \n",
"9-Music video / \n",
"6-Animation / 61-Art / 981-Sculpture / 1420-Gear / \n",
"223-Weather / 419-Earth / 448-Planet / 593-Moon / 654-Sun / 2643-Asteroid / 4478-Sunspot / \n",
"8-Football / \n",
"0-Games / 2-Video game / 19-PC game / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / 858-Carl Johnson / \n",
"6-Animation / 4129-Crayon / \n",
"3-Concert / \n",
"2-Video game / 425-City / 2307-SimCity / 3290-SimCity / \n",
"6-Animation / 15-Cartoon / \n",
"242-Paper / \n",
"3-Concert / \n",
"110-Album / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 272-iPod / 331-iPod touch / 554-iPhone 4 / 1155-iPhone 3GS / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / \n",
"128-Cymbal / \n",
"1-Vehicle / 27-Motorcycle / 158-Tire / \n",
"10-Animal / \n",
"0-Games / 8-Football / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 1084-Bulldog / 1845-Pug / \n",
"6-Animation / 31-Disc jockey / \n",
"1-Vehicle / 82-Airplane / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 366-Dragon / 471-Wing / \n",
"1-Vehicle / 4-Car / 245-BMW / 344-Coupé / \n",
"1-Vehicle / 27-Motorcycle / 67-Cycling / 69-Bicycle / 114-Engine / 806-Chopper / \n",
"22-Nature / 86-Plant / \n",
"2-Video game / 33-Weapon / 96-Soldier / 736-ARMA / 1034-ARMA 2 / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 64-Transport / 74-Truck / \n",
"222-Surfing / 672-Bodyboarding / \n",
"22-Nature / 147-River / 283-Rock / \n",
"5-Dance / 16-Performance art / 95-Talent show / \n",
"0-Games / 2-Video game / 581-PlayStation / 3161-Tarzan / \n",
"7-Musician / 13-Musical ensemble / 44-Drums / 122-Weight training / 203-Marching band / 250-Parade / 820-Bagpipes / 1646-Pipe band / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"121-Photography / 155-Camera / 1855-Video camera / \n",
"25-Toy / 89-Comics / 319-Robot / 435-Action figure / \n",
"886-James Bond / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 172-Motocross / \n",
"14-Guitar / 4101-Lap steel guitar / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 135-Drifting / 1301-Nissan Skyline / \n",
"9-Music video / 96-Soldier / \n",
"0-Games / 23-Mobile phone / 29-Smartphone / 37-Gadget / \n",
"37-Gadget / 47-Personal computer / 228-Laptop / 513-Computer keyboard / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"1-Vehicle / \n",
"130-Gymnastics / \n",
"10-Animal / 540-Insect / 775-Bee / 1296-Nest / 2867-Wasp / \n",
"110-Album / \n",
"3-Concert / 13-Musical ensemble / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 118-Grand Theft Auto V / 330-Money / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 3046-Maltese / \n",
"0-Games / 2-Video game / 241-Fighting game / 1961-Virtua Fighter 5 / 3077-Virtua Fighter / \n",
"5-Dance / 16-Performance art / \n",
"1-Vehicle / 2635-Water well / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 111-PlayStation 3 / 127-Xbox 360 / 134-Call of Duty: Black Ops / 194-Call of Duty: Modern Warfare 2 / \n",
"10-Animal / 80-Horse / \n",
"28-Fashion / 206-Shoe / 333-Nike; Inc. / 368-Sneakers / 4171-Air Force 1 / \n",
"6-Animation / 1206-Cinema 4D / \n",
"86-Plant / 136-Gardening / 156-Agriculture / 174-Farm / 279-Garden / 1720-Grape / 2473-Vineyard / 4644-Vine / \n",
"1-Vehicle / 4-Car / 17-Racing / 41-Sports car / 234-Ford / 352-Need for Speed / 1361-Ford GT / 1600-Need for Speed: World / \n",
"0-Games / 2-Video game / 35-Minecraft / 55-Video game console / 127-Xbox 360 / 181-Xbox / 810-Village / \n",
"148-Painting / 153-Dress / 303-Paint / 476-Textile / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 215-Eating / 274-Meat / 442-Bread / 2313-Bánh / \n",
"6-Animation / 15-Cartoon / 89-Comics / 123-Naruto / 183-Manga / \n",
"28-Fashion / 206-Shoe / 333-Nike; Inc. / 368-Sneakers / \n",
"3-Concert / \n",
"3-Concert / 39-Drums / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 40-Road / 118-Grand Theft Auto V / 127-Xbox 360 / 796-Fire engine / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / 397-Organ / 1666-Hammond organ / \n",
"9-Music video / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 76-Ball / \n",
"9-Music video / \n",
"15-Cartoon / 123-Naruto / 285-Sasuke Uchiha / \n",
"0-Games / 2-Video game / 237-Final Fantasy / 1658-Dissidia Final Fantasy / \n",
"98-Festival / 163-Running / \n",
"0-Games / 8-Football / 76-Ball / \n",
"75-Wrestling / 85-Combat / \n",
"842-Harp / \n",
"1-Vehicle / 87-Boat / 498-Motorboat / \n",
"2297-Elf / \n",
"5-Dance / 16-Performance art / 130-Gymnastics / 350-Circus / 2358-Contortion / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 76-Ball / 81-Athlete / \n",
"0-Games / 745-Writing / 971-Alphabet / \n",
"0-Games / 199-Wii / 960-Badminton / \n",
"10-Animal / 1116-Goat / \n",
"0-Games / 5-Dance / 60-Basketball / \n",
"0-Games / 130-Gymnastics / 1030-Trampoline / 1335-Trampolining / \n",
"1-Vehicle / 4-Car / 1237-Baby carriage / 1932-Child safety seat / \n",
"465-Microphone / \n",
"88-Machine / 261-Tool / 400-Woodturning / 972-Lathe / 2256-Bearing / \n",
"20-Trailer / \n",
"12-Food / 86-Plant / 136-Gardening / 184-Vegetable / 279-Garden / 671-Tomato / 1022-Chili pepper / 1384-Greenhouse / \n",
"117-Boxing / \n",
"1-Vehicle / 4-Car / 74-Truck / 158-Tire / \n",
"3-Concert / 7-Musician / 9-Music video / 13-Musical ensemble / \n",
"6-Animation / 123-Naruto / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 92-Radio-controlled model / 167-Four-wheel drive / 202-Radio-controlled car / 926-Traxxas / \n",
"6-Animation / 15-Cartoon / 126-The Walt Disney Company / 1106-Character / 1266-Death Note / \n",
"6-Animation / 15-Cartoon / 359-Harry Potter (Literary Series) / 3605-Puppetry / \n",
"339-Architecture / 394-Map / 425-City / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"324-Finger / 444-Family / 1599-Bone / \n",
"282-Skin / 2108-Hair removal / \n",
"1-Vehicle / 6-Animation / 449-Honda / 610-Snowmobile / \n",
"18-Outdoor recreation / 59-Winter sport / 112-Ice skating / 316-Figure skating / 632-Ice dancing / \n",
"78-Wedding / \n",
"0-Games / 59-Winter sport / 112-Ice skating / 142-Hockey / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 929-Aerobatics / \n",
"0-Games / 256-Tennis / 974-Racket / 1337-Serve / 1626-Forehand / \n",
"612-Beer / \n",
"9-Music video / 13-Musical ensemble / \n",
"49-School / \n",
"0-Games / 326-Metin2 / \n",
"3-Concert / \n",
"0-Games / 8-Football / 73-Ball / \n",
"3-Concert / \n",
"5-Dance / 2943-KFC / \n",
"402-Jewellery / 829-Bead / 859-Beadwork / 907-Necklace / 1001-Gemstone / 1300-Earrings / \n",
"1172-VHS / \n",
"1000-AdventureQuest Worlds / \n",
"1-Vehicle / 64-Transport / 156-Agriculture / 174-Farm / 660-Rural area / 931-Combine Harvester / \n",
"85-Combat / 693-Sword / 694-Tai chi / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 270-Sedan / 680-Subaru / 1041-Subaru Impreza / 1286-Subaru / 1897-Subaru Impreza WRX / \n",
"93-Comedy / 355-Comedian / \n",
"97-Drawing / 148-Painting / 717-Darth Vader / 4331-Darth Maul / \n",
"93-Comedy / \n",
"1-Vehicle / 4-Car / 114-Engine / 610-Snowmobile / \n",
"122-Weight training / 164-Beach / 214-Muscle / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 184-Vegetable / 210-Cookware and bakeware / 301-Meal / \n",
"1-Vehicle / 2457-Organism / \n",
"8-Football / \n",
"42-Fishing / 423-Coast / 1008-Kite / 4380-Sport kite / \n",
"359-Harry Potter (Literary Series) / \n",
"131-Computer / 141-Microsoft Windows / \n",
"28-Fashion / 78-Wedding / 190-Bride / \n",
"0-Games / 8-Football / \n",
"1-Vehicle / 62-Train / 64-Transport / 152-Track / 3015-Monorail / \n",
"8-Football / 90-Sports game / 180-FIFA 15 / \n",
"3-Concert / \n",
"6-Animation / 15-Cartoon / 816-Gundam / \n",
"0-Games / 35-Minecraft / 488-Door / \n",
"12-Food / 86-Plant / 136-Gardening / 184-Vegetable / 215-Eating / 348-Fruit / 1060-Raw food / \n",
"96-Soldier / \n",
"3-Concert / 31-Disc jockey / \n",
"7-Musician / 46-Choir / 397-Organ / 870-Chapel / \n",
"1-Vehicle / 4-Car / 277-Sport utility vehicle / 515-Dodge / \n",
"1-Vehicle / 4-Car / 167-Four-wheel drive / 198-Off-road vehicle / 360-Jeep / 1205-Jeep Wrangler / \n",
"425-City / \n",
"110-Album / \n",
"18-Outdoor recreation / 67-Cycling / 69-Bicycle / 221-Skatepark / 377-BMX bike / \n",
"12-Food / 26-Cooking / 32-Recipe / 236-Baking / 485-Dough / 560-Sugar / 600-Flour / 1284-Frying / 1576-Doughnut / \n",
"0-Games / 2-Video game / 51-Strategy video game / 2182-Mario Party / \n",
"56-Hair / \n",
"10-Animal / 42-Fishing / 71-Dog / \n",
"1318-Flashlight / \n",
"56-Hair / 106-Hairstyle / 340-Afro-textured hair / \n",
"0-Games / 60-Basketball / \n",
"248-Hotel / 369-Resort / \n",
"6-Animation / 701-Beyblade / \n",
"1-Vehicle / 4-Car / 40-Road / 64-Transport / 74-Truck / 1020-Emergency vehicle / 1168-Neighbourhood / \n",
"0-Games / 434-Ninja / 2169-Ninja World / \n",
"12-Food / \n",
"18-Outdoor recreation / 163-Running / \n",
"246-Furniture / 580-Bedroom / 830-Bed / 1281-Couch / \n",
"31-Disc jockey / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / 1512-Sonic Generations / \n",
"9-Music video / \n",
"4499-Fable II / \n",
"4584-Street Legal Racing: Redline / \n",
"116-Pokémon / 186-Pokémon / \n",
"0-Games / 60-Basketball / 168-Basketball moves / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 70-Driving / \n",
"56-Hair / \n",
"0-Games / 1000-AdventureQuest Worlds / \n",
"9-Music video / \n",
"2625-Gabrielle / \n",
"694-Tai chi / \n",
"222-Surfing / \n",
"14-Guitar / 1231-Mandolin / \n",
"1-Vehicle / 27-Motorcycle / 105-Motorcycling / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / \n",
"3-Concert / 5-Dance / 13-Musical ensemble / \n",
"3-Concert / \n",
"22-Nature / 42-Fishing / 275-Diving / 288-Underwater / 381-Underwater diving / 407-Scuba diving / \n",
"9-Music video / \n",
"6-Animation / 15-Cartoon / 396-One Piece / \n",
"0-Games / 2-Video game / 90-Sports game / 180-FIFA 15 / \n",
"12-Food / 99-Christmas / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 178-Samsung Galaxy / \n",
"222-Surfing / \n",
"3-Concert / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / \n",
"1-Vehicle / 22-Nature / 87-Boat / 92-Radio-controlled model / 225-Lake / \n",
"3-Concert / 16-Performance art / \n",
"56-Hair / 106-Hairstyle / 583-Wig / \n",
"102-Building / 113-House / 170-Home improvement / 276-Room / 339-Architecture / 416-Apartment / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"0-Games / 1576-Doughnut / 3277-The Simpsons: Tapped Out / \n",
"3-Concert / 16-Performance art / \n",
"0-Games / 2-Video game / 311-Street Fighter / 461-Street Fighter IV / 496-Super Street Fighter IV / 563-Ryu / 2468-M. Bison / 2526-Zangief / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 553-Sony Xperia / 4635-Sony Ericsson Xperia X10 Mini / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / 1108-Mushroom / \n",
"78-Wedding / \n",
"1-Vehicle / 27-Motorcycle / 105-Motorcycling / 114-Engine / 154-Wheel / 158-Tire / 207-Exhaust system / 806-Chopper / 2927-Shovel / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"163-Running / 206-Shoe / 368-Sneakers / \n",
"2-Video game / 380-Assassin's Creed / 1048-Assassin's Creed / \n",
"5-Dance / \n",
"66-Bollywood / \n",
"1-Vehicle / 4-Car / 517-Audi / 531-Hatchback / 945-MINI Cooper / 1221-MINI / 4496-Citroën DS3 / \n",
"1-Vehicle / 4-Car / \n",
"0-Games / 2-Video game / 111-PlayStation 3 / 216-Call of Duty: Modern Warfare 3 / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 247-Stallion / 289-Dressage / \n",
"3-Concert / \n",
"10-Animal / 12-Food / 132-Bird / \n",
"1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 2712-Just Cause / \n",
"0-Games / 372-Playing card / 791-Magic: The Gathering / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / \n",
"6-Animation / 15-Cartoon / 123-Naruto / 183-Manga / 285-Sasuke Uchiha / 622-Sakura Haruno / \n",
"93-Comedy / 205-Newscaster / \n",
"7-Musician / 14-Guitar / 36-Piano / 44-Drums / \n",
"10-Animal / 86-Plant / 91-Fish / 258-Aquarium / 2029-Filter / \n",
"83-Skateboarding / 108-Skateboard / 221-Skatepark / 336-Scooter / 595-Champion / \n",
"3-Concert / \n",
"1964-TV4 / \n",
"0-Games / 8-Football / \n",
"20-Trailer / \n",
"0-Games / 1-Vehicle / 74-Truck / 124-Tractor / 174-Farm / 715-Farming Simulator / \n",
"467-Chipmunk / \n",
"8-Football / 79-American football / \n",
"15-Cartoon / 183-Manga / 396-One Piece / \n",
"0-Games / 199-Wii / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / \n",
"0-Games / 2-Video game / 445-Guitar Hero / 724-Rock Band / \n",
"1-Vehicle / 4-Car / 299-Volkswagen Passenger Cars / 827-Pump / \n",
"7-Musician / 9-Music video / 14-Guitar / \n",
"776-Kindergarten / \n",
"0-Games / 8-Football / 54-Highlight film / \n",
"0-Games / 489-Board game / 2833-Go / 4510-Shogi / \n",
"0-Games / 2-Video game / 55-Video game console / 320-Super Smash Bros. / 692-Link / \n",
"783-Zee TV / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"14-Guitar / 842-Harp / 3172-Lyre / \n",
"259-Fire / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"2324-Hang / \n",
"9-Music video / \n",
"1-Vehicle / 62-Train / 64-Transport / 84-Snow / 103-Rail transport / 120-Winter / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 189-iPad / 272-iPod / 331-iPod touch / \n",
"61-Art / 83-Skateboarding / 108-Skateboard / \n",
"0-Games / 2-Video game / 264-Star Wars / \n",
"1-Vehicle / 4-Car / 154-Wheel / 158-Tire / 493-Rim / \n",
"3-Concert / \n",
"3-Concert / 7-Musician / 389-Restaurant / \n",
"0-Games / 2-Video game / 6-Animation / 506-Garry's Mod / \n",
"6-Animation / 15-Cartoon / 89-Comics / \n",
"0-Games / 463-Cricket / 1495-Twenty20 / \n",
"1-Vehicle / 4-Car / 25-Toy / 84-Snow / 92-Radio-controlled model / 202-Radio-controlled car / 449-Honda / 3086-Honda CR-V / \n",
"6-Animation / 4527-South Park: The Stick of Truth / \n",
"12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / \n",
"1-Vehicle / 92-Radio-controlled model / 113-House / 149-Model aircraft / 155-Camera / 160-Radio-controlled aircraft / 393-Unmanned aerial vehicle / 604-Quadcopter / \n",
"6-Animation / 15-Cartoon / 185-Dragon Ball / 244-Goku / 418-Gohan / \n",
"1-Vehicle / 62-Train / 119-Locomotive / 143-Railroad car / 428-Rail transport modelling / \n",
"6-Animation / 15-Cartoon / 89-Comics / 3595-Shaman King / \n",
"12-Food / 86-Plant / 865-Turtle / 1500-Cherry blossom / \n",
"6-Animation / 213-Forest / 1555-Sword Art Online / \n",
"0-Games / 2-Video game / 192-Battlefield / 374-Battlefield 3 / 383-Battlefield 4 / \n",
"1-Vehicle / 4-Car / 87-Boat / 498-Motorboat / 2338-Inflatable boat / \n",
"3-Concert / 30-Drummer / \n",
"0-Games / 2-Video game / 19-PC game / \n",
"3162-Postage stamp / \n",
"1-Vehicle / 4-Car / 114-Engine / 509-Classic car / 1301-Nissan Skyline / \n",
"1-Vehicle / 62-Train / 64-Transport / 161-Train station / \n",
"23-Mobile phone / 101-Telephone / \n",
"47-Personal computer / 131-Computer / 141-Microsoft Windows / \n",
"66-Bollywood / \n",
"14-Guitar / \n",
"3-Concert / 7-Musician / 98-Festival / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 274-Meat / 542-Potato / 968-Pie / 1108-Mushroom / 2650-Frying pan / \n",
"405-Braid / 1249-Rainbow Loom / \n",
"0-Games / 49-School / 81-Athlete / 562-Pitcher / \n",
"22-Nature / 164-Beach / \n",
"2-Video game / 127-Xbox 360 / \n",
"20-Trailer / 42-Fishing / 222-Surfing / 917-Sail / \n",
"10-Animal / 61-Art / 97-Drawing / 148-Painting / 936-Watercolor paint / 1063-Ink / 2630-Doodle / \n",
"0-Games / 2-Video game / 19-PC game / 20-Trailer / 359-Harry Potter (Literary Series) / 4489-Harry Potter and the Philosopher's Stone / \n",
"3-Concert / \n",
"5-Dance / \n",
"28-Fashion / 206-Shoe / 4151-Flip-flops / \n",
"72-iPhone / 83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"6-Animation / 15-Cartoon / 61-Art / 97-Drawing / 183-Manga / 408-Sketch / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"497-Macintosh / 3974-Mac Mini / \n",
"5-Dance / \n",
"1-Vehicle / 4-Car / 20-Trailer / \n",
"0-Games / 2-Video game / 55-Video game console / 317-PlayStation / \n",
"1-Vehicle / 87-Boat / 308-Ship / 579-Yacht / 1957-Luxury yacht / \n",
"1-Vehicle / 4-Car / 11-Motorsport / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"12-Food / 52-Dish / 58-Cuisine / 184-Vegetable / 301-Meal / 391-Chicken meat / 1569-Steamed rice / 3049-Convenience store / \n",
"121-Photography / \n",
"31-Disc jockey / \n",
"122-Weight training / 214-Muscle / \n",
"0-Games / 2-Video game / 55-Video game console / 111-PlayStation 3 / 127-Xbox 360 / 181-Xbox / 317-PlayStation / 386-Xbox / 1482-PlayStation Network / \n",
"130-Gymnastics / \n",
"0-Games / 2-Video game / 55-Video game console / 199-Wii / 774-Galaxy / 1397-Super Mario Galaxy / 1759-Super Mario Galaxy 2 / \n",
"49-School / \n",
"1-Vehicle / 4-Car / 114-Engine / 3399-Timing belt / 3939-Nissan Maxima / \n",
"9-Music video / \n",
"339-Architecture / \n",
"925-Brain / \n",
"61-Art / 97-Drawing / 408-Sketch / 479-Graffiti / 538-Illustration / \n",
"0-Games / 2-Video game / 20-Trailer / 34-Action-adventure game / 111-PlayStation 3 / 176-PlayStation 4 / 656-Metal Gear / 2212-Metal Gear Solid V: The Phantom Pain / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 302-Call of Duty 4: Modern Warfare / \n",
"8-Football / 358-Rugby football / \n",
"122-Weight training / 529-Squat / 532-Barbell / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"9-Music video / 14-Guitar / \n",
"66-Bollywood / \n",
"137-Tree / 1198-Birth / \n",
"1-Vehicle / 9-Music video / 62-Train / 428-Rail transport modelling / \n",
"2642-CNBC / \n",
"5-Dance / \n",
"3-Concert / 5-Dance / \n",
"9-Music video / 99-Christmas / 558-Santa Claus / 964-Christmas tree / \n",
"3-Concert / 16-Performance art / \n",
"3-Concert / \n",
"7-Musician / 14-Guitar / 15-Cartoon / 24-String instrument / 63-Acoustic guitar / \n",
"5-Dance / 31-Disc jockey / 133-Nightclub / \n",
"263-Light / 1375-Laser lighting display / \n",
"20-Trailer / 264-Star Wars / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / 376-Flute / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 19-PC game / 34-Action-adventure game / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 422-Call of Duty: Ghosts / 4485-Warhawk / \n",
"6-Animation / 15-Cartoon / \n",
"7-Musician / 219-Accordion / \n",
"3-Concert / 68-Lighting / 263-Light / \n",
"35-Minecraft / \n",
"59-Winter sport / 81-Athlete / 112-Ice skating / 142-Hockey / 312-Ice rink / \n",
"49-School / \n",
"0-Games / 2-Video game / 6-Animation / 1878-SD Gundam Capsule Fighter / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 1668-Claw crane / \n",
"3-Concert / 7-Musician / \n",
"2-Video game / 429-Website / \n",
"0-Games / 326-Metin2 / \n",
"20-Trailer / 3226-Chucky / \n",
"45-Cosmetics / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 70-Driving / 212-Highway / 231-Supercar / 711-Gran Turismo / 953-Gran Turismo 5 / 1521-Jaguar Cars / \n",
"37-Gadget / 47-Personal computer / 145-Tablet computer / 189-iPad / 1464-iPad Mini / \n",
"0-Games / 8-Football / 1247-Penalty kick / \n",
"0-Games / 83-Skateboarding / 221-Skatepark / 336-Scooter / \n",
"0-Games / 2-Video game / 47-Personal computer / 1028-Devil May Cry / 2344-DmC: Devil May Cry / \n",
"18-Outdoor recreation / 33-Weapon / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 30-Drummer / \n",
"117-Boxing / \n",
"3-Concert / 13-Musical ensemble / 30-Drummer / 133-Nightclub / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 382-Dota 2 / 549-Defense of the Ancients / \n",
"0-Games / 59-Winter sport / 77-Arena / 112-Ice skating / 142-Hockey / 312-Ice rink / \n",
"0-Games / 530-Cue sports / 568-Pool / 994-Cue stick / \n",
"9-Music video / 3124-Kendama / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 84-Snow / 105-Motorcycling / 120-Winter / 262-Ice / 2070-Suzuki Hayabusa / \n",
"49-School / 410-Teacher / \n",
"13-Musical ensemble / \n",
"5-Dance / 49-School / \n",
"702-Counter-Strike: Source / \n",
"9-Music video / 1221-MINI / \n",
"0-Games / 2-Video game / 47-Personal computer / 118-Grand Theft Auto V / 464-First-person Shooter / \n",
"10-Animal / 80-Horse / \n",
"10-Animal / 208-Wood / 540-Insect / 775-Bee / 1296-Nest / \n",
"0-Games / 49-School / 237-Final Fantasy / \n",
"3-Concert / 68-Lighting / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"0-Games / 710-Handball / \n",
"110-Album / \n",
"0-Games / 2-Video game / 60-Basketball / 77-Arena / 90-Sports game / 168-Basketball moves / 255-Slam dunk / 1262-NBA 2K14 / 2156-Rookie / \n",
"10-Animal / 48-Pet / 166-Cat / 469-Bag / 1142-Laundry / \n",
"10-Animal / 48-Pet / 166-Cat / 346-Kitten / \n",
"66-Bollywood / \n",
"1-Vehicle / 74-Truck / 114-Engine / \n",
"3-Concert / 5-Dance / 95-Talent show / 191-Cheerleading / \n",
"5-Dance / 16-Performance art / 133-Nightclub / \n",
"267-Mountain / \n",
"5-Dance / \n",
"616-The Bible / \n",
"1-Vehicle / 4-Car / 261-Tool / 4066-Wrench / \n",
"0-Games / 2-Video game / 116-Pokémon / 186-Pokémon / 1204-Pokémon Ruby and Sapphire / 1665-Pokémon Emerald / 2236-Emerald / \n",
"512-Printing / 696-T-shirt / 1757-Screen printing / \n",
"1-Vehicle / 4-Car / 40-Road / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"14-Guitar / \n",
"1-Vehicle / 18-Outdoor recreation / 22-Nature / 67-Cycling / 69-Bicycle / 197-Trail / 213-Forest / 220-Mountain bike / 278-GoPro / 284-Mountain biking / \n",
"2-Video game / 47-Personal computer / 1045-Computer case / 4515-Antec / \n",
"12-Food / 26-Cooking / 32-Recipe / 226-Dessert / 232-Cake / 295-Chocolate / 1176-Pumpkin / 1707-Microwave oven / 2768-Mug / \n",
"6-Animation / 15-Cartoon / 183-Manga / 185-Dragon Ball / 244-Goku / \n",
"810-Village / \n",
"78-Wedding / \n",
"0-Games / 2-Video game / 11-Motorsport / 2815-F-Zero / \n",
"0-Games / 2-Video game / 37-Gadget / 55-Video game console / 90-Sports game / 292-Handheld game console / 317-PlayStation / 480-FIFA 13 / 682-PlayStation Vita / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 18-Outdoor recreation / 27-Motorcycle / 172-Motocross / \n",
"0-Games / 67-Cycling / 69-Bicycle / 83-Skateboarding / 222-Surfing / 669-Extreme sport / 2014-Vans / \n",
"25-Toy / 89-Comics / 185-Dragon Ball / 244-Goku / 435-Action figure / 1986-Piccolo / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 349-Chevrolet / 378-Drag racing / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 70-Driving / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 161-Train station / 224-Rapid transit / \n",
"0-Games / 2-Video game / 19-PC game / 1566-Doom / \n",
"3-Concert / 5-Dance / 133-Nightclub / 613-Bar / \n",
"1-Vehicle / 4-Car / 40-Road / 120-Winter / 2016-Volvo Cars / \n",
"0-Games / 2851-Fight Night / \n",
"31-Disc jockey / 133-Nightclub / \n",
"18-Outdoor recreation / 217-Hunting / 4035-Pheasant / \n",
"1-Vehicle / 4-Car / 9-Music video / 70-Driving / \n",
"5-Dance / \n",
"10-Animal / 25-Toy / 1152-Plush / 1212-Stuffed toy / 1497-Wool / \n",
"1-Vehicle / 62-Train / 110-Album / \n",
"0-Games / 5-Dance / 49-School / 191-Cheerleading / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 877-Hatsune Miku: Project DIVA / 1648-Hatsune Miku: Project DIVA F / 2682-Hatsune Miku: Project DIVA F 2nd / \n",
"9-Music video / 14-Guitar / 15-Cartoon / 24-String instrument / \n",
"66-Bollywood / \n",
"766-President of the United States / \n",
"0-Games / 2-Video game / 2431-Metal Slug / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"6-Animation / 15-Cartoon / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / \n",
"1-Vehicle / 3-Concert / 11-Motorsport / 27-Motorcycle / 67-Cycling / 69-Bicycle / 350-Circus / \n",
"12-Food / 26-Cooking / 32-Recipe / 179-Kitchen / 184-Vegetable / 1439-Refrigerator / 2614-Lettuce / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"3-Concert / \n",
"93-Comedy / \n",
"5-Dance / 16-Performance art / \n",
"5-Dance / \n",
"1-Vehicle / 11-Motorsport / \n",
"12-Food / 391-Chicken meat / 944-Hamburger / 2943-KFC / \n",
"75-Wrestling / \n",
"5-Dance / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 1028-Devil May Cry / 2336-Devil May Cry 3: Dante's Awakening / \n",
"0-Games / 2-Video game / 19-PC game / 34-Action-adventure game / 43-Call of Duty / 118-Grand Theft Auto V / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / \n",
"1-Vehicle / 11-Motorsport / 135-Drifting / 1887-Tricycle / \n",
"1-Vehicle / 84-Snow / 120-Winter / 223-Weather / 1145-Winter storm / \n",
"318-Church / \n",
"121-Photography / 155-Camera / 566-Digital camera / 629-Camera lens / 721-Digital SLR / 3328-Canon EOS 7D / \n",
"10-Animal / 12-Food / 274-Meat / 763-Snake / \n",
"3-Concert / 5-Dance / \n",
"3-Concert / 7-Musician / 14-Guitar / 30-Drummer / 39-Drums / 44-Drums / \n",
"110-Album / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 297-Grand Theft Auto IV / 658-Grand Theft Auto: The Lost and Damned / \n",
"626-Puzzle / \n",
"0-Games / 2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / \n",
"0-Games / 1-Vehicle / 2-Video game / 124-Tractor / 127-Xbox 360 / 174-Farm / 314-Xbox One / 715-Farming Simulator / \n",
"79-American football / \n",
"1163-Carpet / 2547-Staples Center / \n",
"2-Video game / 55-Video game console / 127-Xbox 360 / 181-Xbox / \n",
"0-Games / 2-Video game / 43-Call of Duty / 111-PlayStation 3 / 127-Xbox 360 / 194-Call of Duty: Modern Warfare 2 / 302-Call of Duty 4: Modern Warfare / \n",
"121-Photography / \n",
"9-Music video / 549-Defense of the Ancients / \n",
"12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / 274-Meat / 620-Barbecue / 688-Beef / 1115-Steak / \n",
"36-Piano / 110-Album / \n",
"0-Games / 2-Video game / 37-Gadget / 55-Video game console / 292-Handheld game console / 317-PlayStation / 362-PlayStation Portable / 682-PlayStation Vita / 2260-Monster Hunter Portable 3rd / \n",
"0-Games / 942-Silkroad Online / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 19-PC game / 1861-Smite / 4525-Hi-Rez Studios / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 184-Vegetable / 236-Baking / 475-Cheese / 485-Dough / 662-Pizza / 671-Tomato / 752-Pasta / 1230-Italian food / 1381-Garlic / 3364-Basil / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 19-PC game / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"10-Animal / 42-Fishing / 91-Fish / 147-River / 225-Lake / 268-Recreational fishing / 896-Largemouth bass / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"1-Vehicle / 27-Motorcycle / 206-Shoe / 757-Boot / \n",
"12-Food / 26-Cooking / 184-Vegetable / 348-Fruit / \n",
"110-Album / 311-Street Fighter / 461-Street Fighter IV / 2239-Street Fighter II: The World Warrior / 2468-M. Bison / 4579-Street Fighter Alpha 3 / \n",
"364-Sitcom / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 172-Motocross / \n",
"3-Concert / \n",
"223-Weather / 259-Fire / 365-Fireworks / \n",
"20-Trailer / 1710-Pirates of the Caribbean / \n",
"257-Counter-Strike / \n",
"273-News program / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"23-Mobile phone / 29-Smartphone / 452-Watch / 627-Clock / 1260-Smartwatch / 3727-Android Wear / \n",
"3-Concert / 5-Dance / 31-Disc jockey / 68-Lighting / 133-Nightclub / \n",
"98-Festival / \n",
"15-Cartoon / 1696-Raven / \n",
"6-Animation / \n",
"0-Games / 779-Point Blank / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 172-Motocross / \n",
"1621-Ork / \n",
"14-Guitar / \n",
"31-Disc jockey / \n",
"9-Music video / 28-Fashion / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 42-Fishing / 91-Fish / 147-River / 225-Lake / 268-Recreational fishing / 478-Fishing rod / \n",
"3-Concert / 7-Musician / 14-Guitar / 39-Drums / \n",
"6-Animation / 20-Trailer / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / \n",
"938-Steel / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"5-Dance / \n",
"1-Vehicle / 4-Car / 74-Truck / 137-Tree / 1093-Loader / 1611-Garbage truck / \n",
"8-Football / \n",
"0-Games / 8-Football / 21-Stadium / 163-Running / \n",
"22-Nature / 1531-Sunrise / \n",
"25-Toy / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 199-Wii / \n",
"372-Playing card / 746-Card manipulation / \n",
"3-Concert / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / \n",
"0-Games / 2-Video game / 1007-Portal / 1274-Portal 2 / \n",
"205-Newscaster / 223-Weather / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 135-Drifting / \n",
"45-Cosmetics / 56-Hair / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 282-Skin / 291-Rouge / 385-Face / 436-Concealer / \n",
"1-Vehicle / 88-Machine / 249-Heavy equipment / 817-Bulldozer / 822-Crane / \n",
"93-Comedy / 355-Comedian / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"12-Food / \n",
"83-Skateboarding / 108-Skateboard / 351-Jumping / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 362-PlayStation Portable / 1125-God of War / 1432-Kratos / \n",
"9-Music video / \n",
"3-Concert / \n",
"2-Video game / 319-Robot / \n",
"1975-Amiga / \n",
"1899-E-book / \n",
"3442-Résumé / \n",
"1-Vehicle / 25-Toy / 62-Train / 1279-Plarail / 1685-Toy train / \n",
"3-Concert / 5-Dance / 16-Performance art / 95-Talent show / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / 30-Drummer / 44-Drums / \n",
"0-Games / 2-Video game / 79-American football / 90-Sports game / 561-Madden NFL / \n",
"1-Vehicle / 12-Food / 666-Concrete / \n",
"18-Outdoor recreation / 59-Winter sport / 112-Ice skating / 703-Roller skating / 1937-Inline skates / 2747-Inline skating / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / 2073-Sabian / \n",
"3-Concert / 16-Performance art / 68-Lighting / \n",
"12-Food / \n",
"20-Trailer / \n",
"0-Games / \n",
"5-Dance / \n",
"23-Mobile phone / 37-Gadget / 47-Personal computer / 145-Tablet computer / 189-iPad / 1459-Magazine / \n",
"7-Musician / 13-Musical ensemble / 38-Orchestra / 250-Parade / 371-Firefighter / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 272-iPod / \n",
"5-Dance / 31-Disc jockey / \n",
"14-Guitar / 36-Piano / 53-Keyboard / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 101-Telephone / 155-Camera / 178-Samsung Galaxy / 899-Samsung Galaxy S III / 3648-Samsung Galaxy S III Mini / \n",
"0-Games / 2-Video game / 116-Pokémon / 186-Pokémon / 1204-Pokémon Ruby and Sapphire / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 650-Convertible / 1455-Cadillac / \n",
"9-Music video / \n",
"5-Dance / 16-Performance art / \n",
"60-Basketball / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"47-Personal computer / 145-Tablet computer / 228-Laptop / 2581-Microsoft Surface / \n",
"468-Biology / 925-Brain / \n",
"12-Food / 26-Cooking / 662-Pizza / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 520-Bicycle frame / 645-Road bicycle / \n",
"1-Vehicle / 266-Helicopter / 3602-Mini-Z / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 2-Video game / 1178-Quest / 1196-Dragon Age / 1735-Dragon Age: Inquisition / 2700-Dragon Age II / \n",
"98-Festival / 253-Carnival / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / 1069-Easter egg / \n",
"3-Concert / 68-Lighting / \n",
"0-Games / 2-Video game / 831-Plants vs. Zombies / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 25-Toy / 40-Road / 57-Race track / 92-Radio-controlled model / 202-Radio-controlled car / 270-Sedan / \n",
"4463-Santana Lopez / \n",
"0-Games / 88-Machine / 535-Slot machine / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"0-Games / 756-Tibia / \n",
"23-Mobile phone / 29-Smartphone / 47-Personal computer / 72-iPhone / 145-Tablet computer / 189-iPad / 272-iPod / 921-Grand Theft Auto: Vice City / \n",
"28-Fashion / 45-Cosmetics / 56-Hair / 583-Wig / 607-Lace / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / 813-Running back / \n",
"6-Animation / 15-Cartoon / 123-Naruto / 801-Inuyasha / \n",
"0-Games / 535-Slot machine / 3153-Freddy Krueger / \n",
"1535-Prize / \n",
"61-Art / 110-Album / \n",
"28-Fashion / 45-Cosmetics / 56-Hair / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 57-Race track / \n",
"12-Food / 26-Cooking / 52-Dish / 58-Cuisine / 215-Eating / 1218-Japanese Cuisine / \n",
"0-Games / 2-Video game / 541-Black-and-white / 1446-Game Boy / \n",
"222-Surfing / 815-Surfboard / \n",
"1-Vehicle / 12-Food / 124-Tractor / 156-Agriculture / 174-Farm / 249-Heavy equipment / 617-Maize / 1473-Forage harvester / \n",
"13-Musical ensemble / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 344-Coupé / 618-Fiat Automobiles / \n",
"0-Games / 159-World of Warcraft / 171-Warcraft / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 49-School / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / \n",
"0-Games / 2-Video game / 23-Mobile phone / 29-Smartphone / 809-Logo / \n",
"155-Camera / \n",
"2-Video game / 34-Action-adventure game / 1396-Lara Croft / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 344-Coupé / 2052-Infiniti / \n",
"61-Art / \n",
"1-Vehicle / 4-Car / 154-Wheel / 158-Tire / 493-Rim / \n",
"0-Games / 2-Video game / 1631-PlanetSide 2 / 2026-PlanetSide / \n",
"85-Combat / 117-Boxing / 699-Kick / \n",
"8-Football / 595-Champion / \n",
"5-Dance / \n",
"5-Dance / 209-University / \n",
"14-Guitar / 24-String instrument / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / \n",
"0-Games / 2-Video game / 851-Digimon / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"66-Bollywood / \n",
"0-Games / 211-RuneScape / 635-Achievement / \n",
"59-Winter sport / 112-Ice skating / 262-Ice / 316-Figure skating / 833-Ice skate / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 2-Video game / 35-Minecraft / 365-Fireworks / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 378-Drag racing / \n",
"98-Festival / \n",
"322-Mixtape / \n",
"8-Football / \n",
"12-Food / 26-Cooking / 32-Recipe / 274-Meat / \n",
"22-Nature / 2493-The Hobbit / \n",
"5-Dance / 115-Ballet / 776-Kindergarten / \n",
"25-Toy / 140-LEGO / 264-Star Wars / 852-Lego Star Wars / \n",
"5-Dance / 4336-Angkor Wat / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 315-Knife / 422-Call of Duty: Ghosts / 4485-Warhawk / \n",
"85-Combat / 1832-Hand-to-hand combat / \n",
"3-Concert / 7-Musician / \n",
"6-Animation / 15-Cartoon / \n",
"9-Music video / 125-Television / \n",
"410-Teacher / 745-Writing / \n",
"67-Cycling / 69-Bicycle / 220-Mountain bike / 284-Mountain biking / 351-Jumping / \n",
"12-Food / 367-Drink / 612-Beer / 3126-Ale / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"3-Concert / 7-Musician / \n",
"28-Fashion / 45-Cosmetics / \n",
"0-Games / 2-Video game / 20-Trailer / 413-PlayStation 2 / 4226-Neo / \n",
"6-Animation / 15-Cartoon / 1732-Oni / \n",
"0-Games / 33-Weapon / 3786-Killing Floor / \n",
"557-Salad / \n",
"1-Vehicle / 18-Outdoor recreation / 22-Nature / 67-Cycling / 69-Bicycle / 197-Trail / 220-Mountain bike / 284-Mountain biking / 3665-Giant Bicycles / \n",
"9-Music video / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"3-Concert / 5-Dance / \n",
"2-Video game / 25-Toy / 229-Halo / \n",
"0-Games / 8-Football / \n",
"325-Vampire / 725-Human / \n",
"0-Games / 162-Medicine / 468-Biology / \n",
"3-Concert / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 139-Airport / \n",
"28-Fashion / 56-Hair / 106-Hairstyle / 405-Braid / \n",
"398-The Legend of Zelda / \n",
"7-Musician / 36-Piano / 150-Violin / 376-Flute / \n",
"2183-Stardoll / \n",
"12-Food / 26-Cooking / 32-Recipe / 179-Kitchen / 367-Drink / 985-Strawberry / 1174-Smoothie / \n",
"125-Television / \n",
"20-Trailer / 1177-Spider / \n",
"925-Brain / \n",
"9-Music video / \n",
"1-Vehicle / 575-Traffic / \n",
"6-Animation / 15-Cartoon / 342-Windows Media Video / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"0-Games / 2-Video game / 257-Counter-Strike / 1109-Counter-Strike Online / \n",
"0-Games / 51-Strategy video game / 373-Clash of Clans / 3067-Boom Beach / \n",
"736-ARMA / 1034-ARMA 2 / \n",
"66-Bollywood / \n",
"7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"22-Nature / 559-Night / 739-Guild Wars / 839-Guild Wars 2 / \n",
"282-Skin / 385-Face / \n",
"28-Fashion / 45-Cosmetics / 56-Hair / 106-Hairstyle / \n",
"3242-Seeley Booth / \n",
"8-Football / 12-Food / 58-Cuisine / 944-Hamburger / \n",
"10-Animal / 22-Nature / 86-Plant / 136-Gardening / 540-Insect / 642-Leaf / \n",
"0-Games / 8-Football / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 680-Subaru / 1286-Subaru / \n",
"1-Vehicle / 4-Car / 118-Grand Theft Auto V / 201-Grand Theft Auto: San Andreas / \n",
"0-Games / 8-Football / 54-Highlight film / \n",
"83-Skateboarding / 108-Skateboard / 154-Wheel / \n",
"2653-Wedding ring / \n",
"6-Animation / 15-Cartoon / 1281-Couch / \n",
"0-Games / 35-Minecraft / 775-Bee / \n",
"0-Games / 1401-The Witcher 3: Wild Hunt / \n",
"0-Games / 51-Strategy video game / 165-League of Legends / \n",
"0-Games / 2-Video game / 19-PC game / 118-Grand Theft Auto V / 192-Battlefield / 383-Battlefield 4 / \n",
"0-Games / 278-GoPro / 463-Cricket / \n",
"110-Album / \n",
"1-Vehicle / 4-Car / 88-Machine / 309-Computer hardware / 602-Central processing unit / \n",
"1-Vehicle / 4-Car / 70-Driving / 94-Dashcam / 212-Highway / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 111-PlayStation 3 / 118-Grand Theft Auto V / \n",
"1-Vehicle / 4-Car / 114-Engine / 154-Wheel / 234-Ford / 918-Brake / 2899-Ford Explorer / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 1221-MINI / 1418-Model car / \n",
"5-Dance / 44-Drums / 98-Festival / 250-Parade / \n",
"0-Games / 2-Video game / 89-Comics / 737-Joker / 1392-Injustice: Gods Among Us / \n",
"29-Smartphone / 1366-LG Optimus series / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 784-Honey / 1022-Chili pepper / \n",
"1-Vehicle / 87-Boat / 92-Radio-controlled model / 342-Windows Media Video / 3311-Hovercraft / \n",
"121-Photography / 155-Camera / 721-Digital SLR / \n",
"6-Animation / 248-Hotel / 369-Resort / 468-Biology / \n",
"3-Concert / 7-Musician / 24-String instrument / 842-Harp / \n",
"5-Dance / 16-Performance art / 115-Ballet / 191-Cheerleading / \n",
"6-Animation / \n",
"593-Moon / 654-Sun / \n",
"6-Animation / 15-Cartoon / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 94-Dashcam / 212-Highway / 2020-Toyota 86 / \n",
"0-Games / 51-Strategy video game / 373-Clash of Clans / \n",
"22-Nature / 164-Beach / \n",
"512-Printing / \n",
"6-Animation / 97-Drawing / 183-Manga / 269-Comic book / 396-One Piece / \n",
"1-Vehicle / 88-Machine / 249-Heavy equipment / 283-Rock / 976-Demolition / 1995-Hammer / \n",
"0-Games / 772-Bowling / 930-Ten-pin bowling / \n",
"3-Concert / 68-Lighting / \n",
"6-Animation / 10-Animal / 15-Cartoon / 80-Horse / 290-Pony / 343-My Little Pony / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 1074-Hyundai / 4199-Hyundai Sonata / \n",
"1-Vehicle / 27-Motorcycle / 154-Wheel / 158-Tire / 918-Brake / 4170-Honda Shadow / \n",
"3-Concert / \n",
"3-Concert / \n",
"14-Guitar / 49-School / 63-Acoustic guitar / \n",
"66-Bollywood / \n",
"1507-Calligraphy / \n",
"3676-Dance Central 3 / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / 858-Carl Johnson / \n",
"6-Animation / 97-Drawing / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / \n",
"6-Animation / 15-Cartoon / 20-Trailer / 185-Dragon Ball / \n",
"2582-Split / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 728-Dirt track racing / \n",
"144-Amusement park / 248-Hotel / 369-Resort / \n",
"2001-Harry Potter and the Deathly Hallows / \n",
"6-Animation / 15-Cartoon / \n",
"10-Animal / 22-Nature / 132-Bird / 280-Wildlife / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 184-Vegetable / 210-Cookware and bakeware / 239-Roasting / 1295-Stir frying / 2959-Wok / \n",
"3-Concert / 68-Lighting / \n",
"0-Games / 2-Video game / 1895-Werewolf / 2054-Diablo II / \n",
"6-Animation / \n",
"5-Dance / 59-Winter sport / 112-Ice skating / 126-The Walt Disney Company / 262-Ice / 316-Figure skating / \n",
"46-Choir / \n",
"110-Album / \n",
"8-Football / \n",
"0-Games / 2-Video game / 19-PC game / 788-Juggling / 4130-Rumble Fighter / \n",
"1-Vehicle / 4-Car / 64-Transport / 371-Firefighter / 796-Fire engine / 1020-Emergency vehicle / \n",
"1-Vehicle / 4-Car / 245-BMW / 4393-BMW 5 Series (F10) / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 47-Personal computer / 72-iPhone / 145-Tablet computer / 189-iPad / 1150-iPad 2 / 1504-iPad 3 / \n",
"23-Mobile phone / 29-Smartphone / \n",
"4359-Anno / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / \n",
"0-Games / 51-Strategy video game / 165-League of Legends / \n",
"8-Football / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 151-Landing / 157-Jet aircraft / 388-Fighter aircraft / \n",
"18-Outdoor recreation / 144-Amusement park / 321-Roller coaster / 334-Amusement ride / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 2375-Ney / \n",
"1-Vehicle / 23-Mobile phone / \n",
"45-Cosmetics / 173-Eye shadow / 1534-H&M / \n",
"1-Vehicle / 4-Car / 64-Transport / 286-Bus / \n",
"3-Concert / 427-Trumpet / \n",
"45-Cosmetics / 276-Room / 402-Jewellery / \n",
"5-Dance / 296-Saxophone / \n",
"5-Dance / 250-Parade / \n",
"365-Fireworks / \n",
"3-Concert / \n",
"31-Disc jockey / \n",
"5-Dance / \n",
"1-Vehicle / 17-Racing / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 163-Running / 220-Mountain bike / 284-Mountain biking / 570-Road bicycle racing / 645-Road bicycle / 674-Marathon / \n",
"18-Outdoor recreation / 22-Nature / 42-Fishing / 85-Combat / 137-Tree / 147-River / 351-Jumping / 1449-Rubber band / 1921-Bungee jumping / \n",
"5-Dance / \n",
"22-Nature / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"5-Dance / 16-Performance art / 115-Ballet / 153-Dress / 1253-Ballet Dancer / \n",
"0-Games / 2-Video game / 19-PC game / 241-Fighting game / 764-The King of Fighters / 2354-The King of Fighters 2002 / \n",
"5-Dance / 78-Wedding / \n",
"3-Concert / 68-Lighting / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / \n",
"44-Drums / \n",
"943-Stove / \n",
"66-Bollywood / \n",
"246-Furniture / 920-Chair / 2669-Leather crafting / 3329-Upholstery / \n",
"1862-Corel / \n",
"126-The Walt Disney Company / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / \n",
"121-Photography / \n",
"3-Concert / 7-Musician / \n",
"6-Animation / 20-Trailer / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"3-Concert / \n",
"3-Concert / 7-Musician / 24-String instrument / \n",
"0-Games / 2-Video game / 55-Video game console / 581-PlayStation / \n",
"89-Comics / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 38-Orchestra / 98-Festival / \n",
"1-Vehicle / 12-Food / 124-Tractor / 156-Agriculture / 174-Farm / 249-Heavy equipment / 931-Combine Harvester / \n",
"3-Concert / 133-Nightclub / 150-Violin / 988-Ibiza / \n",
"1-Vehicle / 84-Snow / 120-Winter / 155-Camera / 522-Helmet / 1323-Helmet camera / \n",
"260-Prayer / \n",
"7-Musician / 24-String instrument / 150-Violin / \n",
"0-Games / 2-Video game / 90-Sports game / 180-FIFA 15 / 561-Madden NFL / \n",
"9-Music video / \n",
"6-Animation / 15-Cartoon / \n",
"110-Album / \n",
"0-Games / 2-Video game / 19-PC game / 1861-Smite / \n",
"3-Concert / 7-Musician / 36-Piano / 107-Musical keyboard / \n",
"7-Musician / 14-Guitar / 24-String instrument / 361-Flamenco / 2380-Flamenco guitar / \n",
"10-Animal / 537-Zoo / 1198-Birth / \n",
"3-Concert / 16-Performance art / \n",
"1-Vehicle / 4-Car / 41-Sports car / 231-Supercar / 569-Lamborghini / 1418-Model car / 2225-Lamborghini Murciélago / \n",
"3-Concert / 7-Musician / \n",
"9-Music video / \n",
"1-Vehicle / 87-Boat / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 516-Headphones / 832-Headset / \n",
"96-Soldier / 99-Christmas / 466-Gift / 1107-United States Navy / \n",
"283-Rock / 1001-Gemstone / \n",
"1-Vehicle / 4-Car / 88-Machine / 114-Engine / \n",
"1-Vehicle / 4-Car / 17-Racing / 57-Race track / 3602-Mini-Z / \n",
"726-Clarinet / \n",
"1-Vehicle / 4-Car / \n",
"23-Mobile phone / 616-The Bible / \n",
"56-Hair / 61-Art / 457-Knitting / 653-Thread / 1536-Woven fabric / \n",
"10-Animal / 80-Horse / 247-Stallion / 289-Dressage / \n",
"1078-Installation art / 1678-Foam / \n",
"49-School / 476-Textile / 1479-Quilt / \n",
"10-Animal / 80-Horse / 247-Stallion / \n",
"1-Vehicle / 4-Car / 552-Honda Civic / \n",
"1-Vehicle / 4-Car / 40-Road / 94-Dashcam / 234-Ford / 473-Ford Mustang / 1285-Parking / \n",
"1-Vehicle / 4-Car / 41-Sports car / \n",
"2-Video game / 192-Battlefield / 374-Battlefield 3 / 2948-Dogfight / 3450-Battlefield 1942 / \n",
"3-Concert / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 291-Rouge / \n",
"155-Camera / 278-GoPro / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / 431-Call of Duty: Advanced Warfare / 1248-Call of Duty: Zombies / \n",
"827-Pump / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 345-Loudspeaker / 1221-MINI / 1565-Logitech / 3285-Boombox / \n",
"3-Concert / 16-Performance art / 46-Choir / \n",
"85-Combat / 117-Boxing / 399-Kickboxing / 1239-K-1 / \n",
"59-Winter sport / 84-Snow / 120-Winter / 177-Skiing / \n",
"18-Outdoor recreation / 163-Running / \n",
"3-Concert / 5-Dance / 13-Musical ensemble / 16-Performance art / 38-Orchestra / \n",
"3-Concert / 7-Musician / 36-Piano / 53-Keyboard / 104-Pianist / \n",
"9-Music video / \n",
"4171-Air Force 1 / \n",
"467-Chipmunk / 846-Squirrel / \n",
"13-Musical ensemble / 820-Bagpipes / 1646-Pipe band / \n",
"3522-ProSieben / \n",
"0-Games / 2-Video game / 43-Call of Duty / 431-Call of Duty: Advanced Warfare / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 64-Transport / 74-Truck / 250-Parade / 371-Firefighter / 796-Fire engine / 1020-Emergency vehicle / \n",
"0-Games / 2-Video game / 77-Arena / 211-RuneScape / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"296-Saxophone / 1006-Tenor saxophone / \n",
"0-Games / 2-Video game / 33-Weapon / 441-The Elder Scrolls / 3418-The Elder Scrolls III: Morrowind / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 30-Drummer / 39-Drums / 44-Drums / \n",
"339-Architecture / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 723-Pig / 1426-Treasure / \n",
"0-Games / 2-Video game / 229-Halo / 644-Halo: Reach / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / \n",
"45-Cosmetics / 235-Lipstick / 1012-Lip gloss / \n",
"414-Computer monitor / \n",
"1-Vehicle / 4-Car / 277-Sport utility vehicle / 1806-GMC / 2683-GMC / \n",
"6-Animation / \n",
"20-Trailer / 2096-The Hobbit / 2493-The Hobbit / 3792-Dwarf / \n",
"341-Sketch comedy / \n",
"10-Animal / 48-Pet / 71-Dog / 1252-Dog agility / \n",
"5-Dance / \n",
"0-Games / 8-Football / 1338-Devil / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 76-Ball / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 118-Grand Theft Auto V / 422-Call of Duty: Ghosts / \n",
"0-Games / 35-Minecraft / 1151-Temple / \n",
"3-Concert / \n",
"28-Fashion / 757-Boot / 1481-Sunglasses / 2326-Royal Air Force / \n",
"9-Music video / \n",
"66-Bollywood / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / \n",
"320-Super Smash Bros. / 335-Nintendo 3DS / 443-Wii U / 697-Super Smash Bros. for Nintendo 3DS and Wii U / 3050-Mewtwo / \n",
"20-Trailer / 294-Batman / \n",
"18-Outdoor recreation / 59-Winter sport / 84-Snow / 120-Winter / 177-Skiing / 305-Ski / \n",
"20-Trailer / \n",
"0-Games / 75-Wrestling / \n",
"283-Rock / 353-Wall / \n",
"12-Food / 26-Cooking / 184-Vegetable / 215-Eating / 944-Hamburger / \n",
"3-Concert / \n",
"6-Animation / 15-Cartoon / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / \n",
"358-Rugby football / \n",
"5-Dance / 16-Performance art / 95-Talent show / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / \n",
"3-Concert / 38-Orchestra / \n",
"3-Concert / 7-Musician / \n",
"1-Vehicle / 87-Boat / 225-Lake / 498-Motorboat / \n",
"12-Food / 32-Recipe / 226-Dessert / 232-Cake / 236-Baking / 295-Chocolate / 578-Cookie / 1322-Biscuit / \n",
"47-Personal computer / 131-Computer / 309-Computer hardware / 957-Desktop computer / 1045-Computer case / \n",
"33-Weapon / 315-Knife / \n",
"5-Dance / 16-Performance art / 115-Ballet / \n",
"35-Minecraft / \n",
"1-Vehicle / 83-Skateboarding / 108-Skateboard / \n",
"9-Music video / \n",
"131-Computer / 189-iPad / \n",
"3-Concert / 5-Dance / 49-School / 130-Gymnastics / 191-Cheerleading / \n",
"1-Vehicle / 250-Parade / \n",
"9-Music video / 594-Music festival / \n",
"25-Toy / 140-LEGO / \n",
"9-Music video / \n",
"0-Games / 8-Football / \n",
"3-Concert / 13-Musical ensemble / 49-School / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / \n",
"155-Camera / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 81-Athlete / 90-Sports game / \n",
"5-Dance / 49-School / \n",
"737-Joker / \n",
"8-Football / 21-Stadium / 60-Basketball / 77-Arena / \n",
"1-Vehicle / 62-Train / 103-Rail transport / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"1-Vehicle / 62-Train / 74-Truck / 2655-Vehicle horn / \n",
"75-Wrestling / 977-WWE '13 / \n",
"67-Cycling / 69-Bicycle / 86-Plant / 136-Gardening / 520-Bicycle frame / \n",
"110-Album / \n",
"3-Concert / \n",
"9-Music video / \n",
"5-Dance / 16-Performance art / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 134-Call of Duty: Black Ops / 216-Call of Duty: Modern Warfare 3 / 302-Call of Duty 4: Modern Warfare / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 2-Video game / 19-PC game / 1412-Half-Life / \n",
"2383-White House / \n",
"1-Vehicle / 4-Car / 11-Motorsport / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 101-Telephone / 527-Battery / 1159-Battery charger / 2163-Rechargeable battery / \n",
"5-Dance / 361-Flamenco / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"1-Vehicle / 4-Car / 212-Highway / 1415-Kia Motors / \n",
"10-Animal / 48-Pet / 91-Fish / 258-Aquarium / \n",
"14-Guitar / 39-Drums / 44-Drums / 521-Metal / \n",
"3-Concert / 68-Lighting / 98-Festival / 3285-Boombox / \n",
"0-Games / 2-Video game / 6-Animation / 849-Tales / 3415-Tales of Vesperia / \n",
"3808-Goggles / \n",
"12-Food / 827-Pump / 1616-Electrical wiring / \n",
"5-Dance / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 184-Vegetable / \n",
"0-Games / 159-World of Warcraft / 171-Warcraft / \n",
"45-Cosmetics / 436-Concealer / \n",
"3-Concert / \n",
"3-Concert / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 57-Race track / 378-Drag racing / \n",
"1-Vehicle / 18-Outdoor recreation / 42-Fishing / 87-Boat / \n",
"28-Fashion / 45-Cosmetics / 56-Hair / 106-Hairstyle / 195-Mascara / 218-Eyelash / 235-Lipstick / 291-Rouge / \n",
"7-Musician / 36-Piano / 53-Keyboard / 104-Pianist / \n",
"3-Concert / \n",
"5-Dance / 16-Performance art / \n",
"0-Games / 2-Video game / 1091-Saints Row / 1695-Saints Row: The Third / \n",
"398-The Legend of Zelda / 541-Black-and-white / 1228-The Legend of Zelda: Ocarina of Time / \n",
"6-Animation / 15-Cartoon / 4276-Toaster / \n",
"2532-Pan flute / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 382-Dota 2 / 549-Defense of the Ancients / 1378-Multiplayer online battle arena / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 230-Brass instrument / \n",
"5-Dance / \n",
"810-Village / \n",
"0-Games / 211-RuneScape / \n",
"98-Festival / 102-Building / 2341-Fine art / \n",
"6-Animation / 15-Cartoon / \n",
"1-Vehicle / 4-Car / 154-Wheel / 234-Ford / 918-Brake / 2256-Bearing / 3684-Ford Bronco / \n",
"12-Food / 3117-Match / \n",
"5-Dance / \n",
"309-Computer hardware / \n",
"0-Games / 35-Minecraft / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"6-Animation / \n",
"56-Hair / 99-Christmas / 106-Hairstyle / 405-Braid / \n",
"2799-Ys / \n",
"46-Choir / \n",
"25-Toy / \n",
"6-Animation / 34-Action-adventure game / 264-Star Wars / \n",
"1880-Marinera / \n",
"1249-Rainbow Loom / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / 318-Church / \n",
"0-Games / 211-RuneScape / 1178-Quest / \n",
"3-Concert / 5-Dance / 153-Dress / \n",
"1-Vehicle / 74-Truck / \n",
"8-Football / \n",
"12-Food / 26-Cooking / 32-Recipe / 179-Kitchen / \n",
"14-Guitar / 24-String instrument / \n",
"18-Outdoor recreation / 163-Running / 674-Marathon / \n",
"0-Games / 2-Video game / 33-Weapon / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 216-Call of Duty: Modern Warfare 3 / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / 44-Drums / \n",
"5-Dance / 16-Performance art / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 608-Wheelie / 927-Supermoto / \n",
"0-Games / 772-Bowling / 930-Ten-pin bowling / \n",
"0-Games / \n",
"12-Food / 612-Beer / 720-Wine / \n",
"6-Animation / 547-Bleach / \n",
"1-Vehicle / 4-Car / 901-Mitsubishi / 1461-Electric car / 2304-Electric vehicle / \n",
"144-Amusement park / 248-Hotel / 369-Resort / \n",
"3-Concert / 5-Dance / 16-Performance art / 95-Talent show / \n",
"6-Animation / 15-Cartoon / 89-Comics / 183-Manga / 185-Dragon Ball / 244-Goku / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / \n",
"0-Games / 59-Winter sport / 112-Ice skating / 142-Hockey / 312-Ice rink / \n",
"0-Games / 523-Cattle / 626-Puzzle / \n",
"61-Art / 587-Clay / 970-Monkey / \n",
"5-Dance / 44-Drums / \n",
"0-Games / 8-Football / 21-Stadium / 77-Arena / \n",
"86-Plant / \n",
"3-Concert / 16-Performance art / 46-Choir / \n",
"1-Vehicle / 4-Car / 207-Exhaust system / 299-Volkswagen Passenger Cars / 1761-Volkswagen Jetta / 3155-Volkswagen Golf Mk4 / \n",
"25-Toy / 626-Puzzle / 631-Cube / 740-Rubik's Cube / \n",
"0-Games / 19-PC game / 159-World of Warcraft / 171-Warcraft / \n",
"6-Animation / 9-Music video / 97-Drawing / 408-Sketch / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 77-Arena / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"0-Games / 8-Football / 110-Album / \n",
"7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"5-Dance / 36-Piano / 53-Keyboard / 763-Snake / \n",
"0-Games / 1-Vehicle / 50-Aircraft / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 266-Helicopter / 392-Radio-controlled helicopter / \n",
"6-Animation / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 230-Brass instrument / \n",
"3-Concert / 46-Choir / 49-School / 776-Kindergarten / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"31-Disc jockey / \n",
"3-Concert / 7-Musician / 16-Performance art / 98-Festival / 594-Music festival / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 1505-Ultimate Marvel vs. Capcom 3 / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"22-Nature / \n",
"28-Fashion / 153-Dress / 390-Model / 495-Runway / \n",
"1-Vehicle / 4-Car / 41-Sports car / 231-Supercar / 569-Lamborghini / \n",
"25-Toy / \n",
"208-Wood / 981-Sculpture / 1346-Wood carving / 1480-Astronaut / \n",
"222-Surfing / 233-Ocean / \n",
"0-Games / 2-Video game / 51-Strategy video game / 356-Star / 373-Clash of Clans / \n",
"56-Hair / 123-Naruto / 583-Wig / \n",
"3-Concert / 16-Performance art / \n",
"8-Football / 9-Music video / \n",
"12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / 3288-Mustard / \n",
"5-Dance / \n",
"12-Food / 786-Juice / 1081-Lemon / 3598-Lime / \n",
"12-Food / 32-Recipe / 2588-Mayonnaise / \n",
"122-Weight training / 204-Gym / 214-Muscle / \n",
"15-Cartoon / 185-Dragon Ball / 244-Goku / 418-Gohan / \n",
"0-Games / 2-Video game / 6-Animation / 514-Tekken / 797-Tekken / 1470-Tekken Tag Tournament 2 / \n",
"30-Drummer / 39-Drums / 44-Drums / 1953-Roland V-Drums / \n",
"9-Music video / 66-Bollywood / \n",
"2-Video game / \n",
"25-Toy / 75-Wrestling / 164-Beach / 435-Action figure / \n",
"6-Animation / 439-Barbie / \n",
"31-Disc jockey / \n",
"3-Concert / 7-Musician / 16-Performance art / 30-Drummer / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"12-Food / 617-Maize / \n",
"46-Choir / 99-Christmas / \n",
"1-Vehicle / 4-Car / 154-Wheel / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 1681-Mu Online / \n",
"45-Cosmetics / 466-Gift / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 96-Soldier / 257-Counter-Strike / \n",
"8-Football / 358-Rugby football / \n",
"130-Gymnastics / 1417-Balance beam / 2745-Beam / \n",
"1621-Ork / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / 30-Drummer / 39-Drums / \n",
"0-Games / 379-Touhou Project / 3849-Perfect Cherry Blossom / \n",
"6-Animation / 3588-Berserk / \n",
"1-Vehicle / 283-Rock / 1409-Missile / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 70-Driving / 207-Exhaust system / 270-Sedan / 344-Coupé / 349-Chevrolet / 406-Chevrolet / 947-Muffler / 1752-Chevrolet SS / \n",
"6-Animation / 123-Naruto / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 247-Stallion / 438-Mare / 3383-Filly / \n",
"163-Running / \n",
"89-Comics / \n",
"12-Food / \n",
"66-Bollywood / \n",
"3-Concert / 68-Lighting / 77-Arena / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 274-Meat / \n",
"12-Food / \n",
"6-Animation / 185-Dragon Ball / 244-Goku / \n",
"12-Food / 424-Chicken / 597-Soup / \n",
"1-Vehicle / 4-Car / 1194-Automotive lighting / \n",
"6-Animation / 15-Cartoon / 61-Art / 251-Sonic the Hedgehog / \n",
"9-Music video / 3401-Aggro Berlin / \n",
"23-Mobile phone / 29-Smartphone / 101-Telephone / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 194-Call of Duty: Modern Warfare 2 / \n",
"5-Dance / 31-Disc jockey / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / 237-Final Fantasy / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 35-Minecraft / 1007-Portal / \n",
"61-Art / 238-Nail / 300-Nail art / 338-Manicure / \n",
"1-Vehicle / 4-Car / 1134-Taxicab / \n",
"9-Music video / \n",
"5-Dance / 66-Bollywood / \n",
"3-Concert / 68-Lighting / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 70-Driving / 207-Exhaust system / 231-Supercar / \n",
"7-Musician / 13-Musical ensemble / 98-Festival / 203-Marching band / \n",
"18-Outdoor recreation / 137-Tree / 148-Painting / 303-Paint / 2124-Spray painting / 2853-Aerosol paint / 3188-Camouflage / \n",
"0-Games / 1991-Rappelz / \n",
"1-Vehicle / 2-Video game / 4-Car / 1333-Wangan Midnight / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 556-Skateboarding trick / \n",
"3-Concert / 31-Disc jockey / 133-Nightclub / \n",
"12-Food / 184-Vegetable / 348-Fruit / 671-Tomato / 1193-Orange / 2676-Citrus / 3207-Peel / 3293-Vegetable carving / \n",
"23-Mobile phone / 29-Smartphone / \n",
"5-Dance / 6-Animation / 16-Performance art / 126-The Walt Disney Company / 576-Walt Disney World / 1139-Belle / 1265-Kyle Kingson / \n",
"0-Games / 372-Playing card / 791-Magic: The Gathering / 1784-Goblin / \n",
"9-Music video / \n",
"1338-Devil / \n",
"1-Vehicle / 4-Car / 70-Driving / 94-Dashcam / \n",
"0-Games / 2-Video game / 453-Pro Evolution Soccer / 3526-Pro Evolution Soccer 2009 / \n",
"20-Trailer / 66-Bollywood / \n",
"5-Dance / 59-Winter sport / 112-Ice skating / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / 1936-Bouzouki / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 2-Video game / 8-Football / 54-Highlight film / 90-Sports game / 111-PlayStation 3 / 453-Pro Evolution Soccer / \n",
"31-Disc jockey / \n",
"61-Art / 148-Painting / \n",
"1-Vehicle / 4-Car / 40-Road / 94-Dashcam / 2850-Chauffeur / \n",
"28-Fashion / 56-Hair / 106-Hairstyle / 340-Afro-textured hair / 919-Hair conditioner / \n",
"56-Hair / \n",
"0-Games / 8-Football / 90-Sports game / 453-Pro Evolution Soccer / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 216-Call of Duty: Modern Warfare 3 / \n",
"3214-Black Widow / \n",
"0-Games / 710-Handball / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"0-Games / 10-Animal / 51-Strategy video game / 550-Dinosaur / 1718-Spore / \n",
"0-Games / 75-Wrestling / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"1-Vehicle / 4-Car / 70-Driving / 1321-Škoda Auto / 2690-Škoda Octavia / \n",
"0-Games / 8-Football / 79-American football / \n",
"5-Dance / \n",
"31-Disc jockey / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 86-Plant / 137-Tree / 167-Four-wheel drive / 198-Off-road vehicle / 360-Jeep / 565-Mud bogging / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"31-Disc jockey / \n",
"1-Vehicle / 22-Nature / 50-Aircraft / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 266-Helicopter / 392-Radio-controlled helicopter / \n",
"9-Music video / 14-Guitar / 24-String instrument / 63-Acoustic guitar / 4484-Nikon D7000 / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 220-Mountain bike / 3611-Folding bicycle / \n",
"45-Cosmetics / \n",
"37-Gadget / 832-Headset / \n",
"10-Animal / 42-Fishing / 91-Fish / 275-Diving / 288-Underwater / 381-Underwater diving / \n",
"0-Games / 2-Video game / 159-World of Warcraft / 171-Warcraft / \n",
"3-Concert / 7-Musician / 38-Orchestra / 219-Accordion / \n",
"12-Food / 612-Beer / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 853-Dune buggy / \n",
"0-Games / 60-Basketball / 77-Arena / 168-Basketball moves / \n",
"208-Wood / 303-Paint / 400-Woodturning / 462-Table / 673-Saw / 2548-Table saw / \n",
"7-Musician / 13-Musical ensemble / 38-Orchestra / 150-Violin / 230-Brass instrument / 1574-French horn / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 24-String instrument / 670-Banjo / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 96-Soldier / 194-Call of Duty: Modern Warfare 2 / \n",
"141-Microsoft Windows / \n",
"1-Vehicle / 6-Animation / 50-Aircraft / 82-Airplane / 129-Aviation / \n",
"49-School / 99-Christmas / \n",
"60-Basketball / 168-Basketball moves / \n",
"5-Dance / 889-Swing / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 161-Train station / \n",
"6-Animation / 15-Cartoon / 477-Transformers / 2186-Megatron / 3398-Transformers: Generation 1 / \n",
"10-Animal / 48-Pet / 166-Cat / 346-Kitten / \n",
"0-Games / 35-Minecraft / 3331-Quarry / \n",
"12-Food / 26-Cooking / 32-Recipe / 36-Piano / 53-Keyboard / 109-Cooking show / 215-Eating / 236-Baking / \n",
"8-Football / \n",
"61-Art / 447-Sand / 3282-Sand art and play / 3550-Sand animation / \n",
"5-Dance / 78-Wedding / 259-Fire / \n",
"8-Football / \n",
"102-Building / 113-House / 170-Home improvement / 276-Room / 580-Bedroom / \n",
"5-Dance / 16-Performance art / 115-Ballet / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / 318-Church / 870-Chapel / \n",
"18-Outdoor recreation / 42-Fishing / 548-Parachuting / 573-Paragliding / 769-Red Bull / \n",
"3822-Mold / \n",
"38-Orchestra / \n",
"497-Macintosh / 819-World Wide Web / \n",
"6-Animation / \n",
"5-Dance / \n",
"170-Home improvement / 652-Bathroom / \n",
"6-Animation / 20-Trailer / \n",
"1-Vehicle / 4-Car / 88-Machine / 114-Engine / \n",
"1-Vehicle / 18-Outdoor recreation / \n",
"1-Vehicle / 11-Motorsport / 172-Motocross / \n",
"3-Concert / 708-Mermaid / \n",
"0-Games / 2-Video game / 43-Call of Duty / 704-Call of Duty: World at War / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 76-Ball / 77-Arena / 81-Athlete / \n",
"8-Football / 696-T-shirt / 2693-Puma SE / \n",
"0-Games / 2-Video game / 33-Weapon / 96-Soldier / 257-Counter-Strike / \n",
"13-Musical ensemble / 46-Choir / \n",
"28-Fashion / 45-Cosmetics / 676-Handbag / 785-Jeans / \n",
"2-Video game / 141-Microsoft Windows / 2302-Dying Light / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"840-Museum / \n",
"6-Animation / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"356-Star / \n",
"37-Gadget / 47-Personal computer / 309-Computer hardware / 700-Hard disk drive / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"47-Personal computer / 58-Cuisine / 145-Tablet computer / \n",
"162-Medicine / \n",
"162-Medicine / 209-University / \n",
"163-Running / \n",
"1-Vehicle / 59-Winter sport / 84-Snow / 120-Winter / 177-Skiing / 305-Ski / \n",
"6-Animation / \n",
"9-Music video / \n",
"42-Fishing / 825-Stream / \n",
"136-Gardening / 585-Lawn / 2437-String trimmer / \n",
"1-Vehicle / 18-Outdoor recreation / 22-Nature / 40-Road / 67-Cycling / 69-Bicycle / 86-Plant / 137-Tree / 197-Trail / 213-Forest / 1887-Tricycle / 3713-Recumbent bicycle / \n",
"3-Concert / 7-Musician / \n",
"1264-Head / \n",
"28-Fashion / 45-Cosmetics / 56-Hair / \n",
"36-Piano / 150-Violin / 401-Cello / \n",
"1-Vehicle / 27-Motorcycle / 2242-Pillow / \n",
"2887-Swing / \n",
"3-Concert / 5-Dance / 16-Performance art / 38-Orchestra / 68-Lighting / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 155-Camera / 566-Digital camera / \n",
"481-Kayak / 949-Rowing / 1090-Canoe / 1305-Canoeing / 1372-Paddle / 3858-Canoe Slalom / \n",
"0-Games / 2-Video game / 47-Personal computer / 456-Go-kart / 518-Mario Kart / 2901-Mario Kart 64 / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 382-Dota 2 / 549-Defense of the Ancients / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 247-Stallion / 289-Dressage / 351-Jumping / 404-Livestock / \n",
"83-Skateboarding / 1105-Fingerboard / \n",
"28-Fashion / 153-Dress / 2392-Blouse / \n",
"209-University / 3503-Doctor / \n",
"3-Concert / \n",
"22-Nature / \n",
"0-Games / 2-Video game / 20-Trailer / 379-Touhou Project / 518-Mario Kart / \n",
"1-Vehicle / 18-Outdoor recreation / 86-Plant / 124-Tractor / 156-Agriculture / 174-Farm / 252-Tractor pulling / \n",
"0-Games / 6-Animation / 15-Cartoon / 323-Toddler / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / \n",
"2-Video game / 243-Sonic the Hedgehog / \n",
"9-Music video / \n",
"9-Music video / \n",
"5-Dance / 350-Circus / \n",
"242-Paper / 370-Origami / 466-Gift / 469-Bag / \n",
"2-Video game / 20-Trailer / 426-Resident Evil / \n",
"419-Earth / \n",
"18-Outdoor recreation / 163-Running / 674-Marathon / \n",
"1-Vehicle / 4-Car / 2616-Chevrolet S-10 / \n",
"812-Plastic / 2913-Drawer / \n",
"0-Games / 2-Video game / 19-PC game / 47-Personal computer / 192-Battlefield / 374-Battlefield 3 / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 154-Wheel / 234-Ford / 349-Chevrolet / 406-Chevrolet / 473-Ford Mustang / 647-Muscle car / 761-Chevrolet Camaro / 952-Chevrolet Corvette / \n",
"10-Animal / 71-Dog / \n",
"28-Fashion / 45-Cosmetics / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / 30-Drummer / 39-Drums / 44-Drums / \n",
"159-World of Warcraft / \n",
"10-Animal / 22-Nature / 155-Camera / 280-Wildlife / 846-Squirrel / 4388-Canon EOS 650D / \n",
"122-Weight training / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 67-Cycling / 69-Bicycle / 2133-Road racing / \n",
"12-Food / 18-Outdoor recreation / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / \n",
"0-Games / 2-Video game / 4-Car / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"5-Dance / \n",
"293-Slide show / 444-Family / \n",
"6-Animation / 97-Drawing / 148-Painting / 3411-Digital painting / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 135-Drifting / \n",
"504-Web page / 819-World Wide Web / \n",
"318-Church / \n",
"1-Vehicle / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 266-Helicopter / 392-Radio-controlled helicopter / \n",
"79-American football / 122-Weight training / 204-Gym / 532-Barbell / \n",
"3-Concert / 68-Lighting / \n",
"3-Concert / 16-Performance art / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 70-Driving / 2518-Dirt 3 / \n",
"342-Windows Media Video / 548-Parachuting / 4334-Paratrooper / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 8-Football / 76-Ball / 90-Sports game / 453-Pro Evolution Soccer / 1719-Pro Evolution Soccer 2015 / \n",
"1794-Curtain / \n",
"0-Games / 8-Football / 79-American football / \n",
"15-Cartoon / 123-Naruto / 183-Manga / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 42-Fishing / 91-Fish / 132-Bird / 147-River / 268-Recreational fishing / 478-Fishing rod / \n",
"31-Disc jockey / \n",
"110-Album / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 1763-Lotus Cars / \n",
"23-Mobile phone / 29-Smartphone / 47-Personal computer / 72-iPhone / 131-Computer / \n",
"102-Building / 248-Hotel / \n",
"1-Vehicle / 85-Combat / 259-Fire / 262-Ice / 371-Firefighter / 1245-Alarm device / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 172-Motocross / \n",
"7-Musician / 14-Guitar / 24-String instrument / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / \n",
"661-Stretching / \n",
"28-Fashion / 206-Shoe / 333-Nike; Inc. / \n",
"273-News program / \n",
"1-Vehicle / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 393-Unmanned aerial vehicle / 604-Quadcopter / \n",
"5-Dance / 430-Ballroom dance / 678-Latin dance / 889-Swing / \n",
"0-Games / 5-Dance / 79-American football / 191-Cheerleading / \n",
"410-Teacher / \n",
"117-Boxing / \n",
"59-Winter sport / 112-Ice skating / 142-Hockey / \n",
"9-Music video / \n",
"9-Music video / \n",
"689-Harmonica / 810-Village / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 76-Ball / 77-Arena / \n",
"9-Music video / 347-Climbing / 2080-Cup / 3686-The North Face / \n",
"1-Vehicle / 4-Car / 3679-Super GT / 4612-Sepang International Circuit / \n",
"12-Food / 2397-Cabbage / \n",
"1-Vehicle / 27-Motorcycle / 105-Motorcycling / \n",
"8-Football / \n",
"5-Dance / 16-Performance art / 769-Red Bull / \n",
"2077-Chemical reaction / \n",
"0-Games / 2-Video game / 159-World of Warcraft / 171-Warcraft / \n",
"0-Games / 8-Football / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 74-Truck / 375-Pickup truck / 2683-GMC / \n",
"0-Games / 2-Video game / 803-DC Comics / 1515-DC Universe Online / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / \n",
"10-Animal / 48-Pet / 71-Dog / 1084-Bulldog / 2438-French Bulldog / \n",
"3-Concert / 7-Musician / 16-Performance art / 38-Orchestra / \n",
"89-Comics / 396-One Piece / 749-Figurine / \n",
"0-Games / 2-Video game / 217-Hunting / 596-Monster Hunter / 2181-Monster Hunter 4 / \n",
"8-Football / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"66-Bollywood / \n",
"5-Dance / 49-School / 191-Cheerleading / \n",
"0-Games / 2-Video game / 229-Halo / 470-Halo 3 / 2002-Halo 3: ODST / \n",
"0-Games / 2-Video game / 33-Weapon / 96-Soldier / 192-Battlefield / 383-Battlefield 4 / \n",
"0-Games / 2-Video game / 33-Weapon / 1394-Alliance of Valiant Arms / \n",
"5-Dance / \n",
"123-Naruto / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 134-Call of Duty: Black Ops / 216-Call of Duty: Modern Warfare 3 / \n",
"1-Vehicle / \n",
"0-Games / 1-Vehicle / 2-Video game / 19-PC game / 33-Weapon / 96-Soldier / 734-DayZ / 736-ARMA / 1034-ARMA 2 / \n",
"0-Games / 54-Highlight film / 60-Basketball / 168-Basketball moves / \n",
"372-Playing card / 746-Card manipulation / \n",
"533-Oil / 536-Glass / 1510-Liquid / 4325-Detergent / \n",
"9-Music video / \n",
"0-Games / 379-Touhou Project / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 184-Vegetable / 688-Beef / \n",
"5-Dance / 98-Festival / \n",
"10-Animal / 166-Cat / 970-Monkey / \n",
"1004-Samurai / \n",
"0-Games / 19-PC game / 51-Strategy video game / 165-League of Legends / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 237-Final Fantasy / \n",
"6-Animation / 15-Cartoon / 251-Sonic the Hedgehog / \n",
"4175-8 mm film / \n",
"23-Mobile phone / 29-Smartphone / 932-Microsoft Lumia / \n",
"196-Eye / 402-Jewellery / 2167-Pendant / \n",
"6-Animation / 15-Cartoon / 379-Touhou Project / \n",
"12-Food / 720-Wine / 1720-Grape / 4470-Winery / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / 265-Electronic keyboard / 1909-Analog synthesizer / \n",
"75-Wrestling / 1042-Jumbotron / \n",
"31-Disc jockey / \n",
"31-Disc jockey / \n",
"133-Nightclub / \n",
"10-Animal / 20-Trailer / 48-Pet / 71-Dog / 298-Television advertisement / \n",
"457-Knitting / 546-Stitch / 653-Thread / \n",
"10-Animal / 25-Toy / 71-Dog / 319-Robot / \n",
"5-Dance / 16-Performance art / 3831-Supreme / \n",
"83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"56-Hair / 607-Lace / \n",
"47-Personal computer / 309-Computer hardware / 513-Computer keyboard / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / \n",
"530-Cue sports / 568-Pool / \n",
"1-Vehicle / 4-Car / 74-Truck / 259-Fire / 371-Firefighter / 2414-Firefighting / \n",
"45-Cosmetics / \n",
"1601-Social media / \n",
"2-Video game / 961-Sega Genesis / 1135-Dolphin / 2638-Master System / 3980-Sega CD / \n",
"1-Vehicle / 2-Video game / 20-Trailer / 34-Action-adventure game / 118-Grand Theft Auto V / 297-Grand Theft Auto IV / 658-Grand Theft Auto: The Lost and Damned / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 597-Soup / 2234-Avocado / \n",
"8-Football / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 4128-Nokia N97 / \n",
"5-Dance / 1240-Mambo / \n",
"0-Games / 326-Metin2 / \n",
"33-Weapon / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"22-Nature / \n",
"905-Flash Video / 2808-Printed circuit board / \n",
"37-Gadget / 47-Personal computer / 228-Laptop / 414-Computer monitor / \n",
"20-Trailer / 298-Television advertisement / 1059-Movie theater / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"3-Concert / \n",
"22-Nature / \n",
"246-Furniture / 251-Sonic the Hedgehog / 276-Room / \n",
"0-Games / 10-Animal / 384-Horse racing / \n",
"2532-Pan flute / \n",
"8-Football / 9-Music video / 76-Ball / \n",
"5-Dance / 892-Aerobics / \n",
"8-Football / \n",
"5-Dance / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 761-Chevrolet Camaro / 947-Muffler / \n",
"5-Dance / 95-Talent show / \n",
"68-Lighting / 121-Photography / 721-Digital SLR / \n",
"31-Disc jockey / 2193-CDJ / \n",
"93-Comedy / \n",
"0-Games / 8-Football / 21-Stadium / 81-Athlete / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / \n",
"6-Animation / 15-Cartoon / 851-Digimon / \n",
"3-Concert / \n",
"1-Vehicle / 88-Machine / 363-Soil / \n",
"78-Wedding / \n",
"0-Games / 2-Video game / 19-PC game / 34-Action-adventure game / 35-Minecraft / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / 689-Harmonica / \n",
"10-Animal / 86-Plant / 91-Fish / 184-Vegetable / 258-Aquarium / 1258-Goldfish / \n",
"8-Football / 110-Album / \n",
"9-Music video / \n",
"0-Games / 19-PC game / 35-Minecraft / \n",
"3-Concert / 16-Performance art / \n",
"9-Music video / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 42-Fishing / 72-iPhone / 189-iPad / 730-Final Fantasy VII / \n",
"1222-Fountain / \n",
"1-Vehicle / 4-Car / 70-Driving / \n",
"5-Dance / 16-Performance art / \n",
"5-Dance / 16-Performance art / 95-Talent show / \n",
"122-Weight training / 204-Gym / \n",
"111-PlayStation 3 / 243-Sonic the Hedgehog / 4019-Sonic the Hedgehog CD / \n",
"12-Food / 122-Weight training / 214-Muscle / 215-Eating / \n",
"10-Animal / 48-Pet / 132-Bird / 598-Parrot / \n",
"6-Animation / \n",
"242-Paper / 370-Origami / \n",
"0-Games / 47-Personal computer / 504-Web page / \n",
"75-Wrestling / \n",
"0-Games / 20-Trailer / 33-Weapon / 3708-Dead Space 3 / \n",
"0-Games / 2-Video game / 686-Diablo III / 872-Diablo / \n",
"14-Guitar / 24-String instrument / 1063-Ink / \n",
"271-Tank / \n",
"946-Baseball park / \n",
"3-Concert / 5-Dance / 7-Musician / 13-Musical ensemble / 46-Choir / 98-Festival / 318-Church / \n",
"6-Animation / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 114-Engine / 1036-Moped / \n",
"3-Concert / 13-Musical ensemble / \n",
"0-Games / 2-Video game / 192-Battlefield / 383-Battlefield 4 / \n",
"150-Violin / 545-Quartet (ensemble) / 1002-String quartet / \n",
"93-Comedy / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 94-Dashcam / 155-Camera / 212-Highway / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 124-Tractor / 363-Soil / 459-Plough / \n",
"6-Animation / 123-Naruto / \n",
"110-Album / \n",
"0-Games / 2-Video game / 464-First-person Shooter / \n",
"0-Games / 2-Video game / 2662-Brave Frontier / \n",
"3-Concert / \n",
"429-Website / 504-Web page / \n",
"3-Concert / 7-Musician / 16-Performance art / 465-Microphone / \n",
"3-Concert / 133-Nightclub / \n",
"3-Concert / 13-Musical ensemble / \n",
"0-Games / 2-Video game / 1027-Terraria / 1149-Castle / \n",
"227-Water / 287-Human swimming / 329-Swimming pool / \n",
"3-Concert / 68-Lighting / 365-Fireworks / \n",
"28-Fashion / 206-Shoe / \n",
"0-Games / 5-Dance / \n",
"1-Vehicle / 64-Transport / 124-Tractor / 174-Farm / 278-GoPro / 2218-Manure / \n",
"0-Games / 2-Video game / 3664-Super Mario 3D Land / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 30-Drummer / 39-Drums / \n",
"1-Vehicle / 4-Car / 245-BMW / 492-Floor / \n",
"1-Vehicle / 666-Concrete / 1869-Cement / \n",
"1-Vehicle / 33-Weapon / 96-Soldier / 271-Tank / \n",
"3-Concert / 7-Musician / \n",
"6-Animation / 25-Toy / 269-Comic book / 701-Beyblade / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 135-Drifting / 169-Rallying / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 477-Transformers / 990-Optimus Prime / 2186-Megatron / 2364-Cybertron / 3759-Transformers: Fall of Cybertron / \n",
"1-Vehicle / 4-Car / 41-Sports car / 449-Honda / 552-Honda Civic / \n",
"3-Concert / 2032-Pepsi / \n",
"3-Concert / \n",
"1-Vehicle / 22-Nature / 573-Paragliding / 1013-Walking / \n",
"60-Basketball / \n",
"9-Music video / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 428-Rail transport modelling / 2935-Garden railway / \n",
"1791-Stargate / \n",
"0-Games / 2-Video game / 2348-Chrono Trigger / \n",
"0-Games / 774-Galaxy / \n",
"85-Combat / 117-Boxing / \n",
"0-Games / 8-Football / \n",
"3-Concert / 5-Dance / 153-Dress / \n",
"0-Games / 60-Basketball / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"72-iPhone / \n",
"12-Food / 582-Candy / \n",
"117-Boxing / \n",
"1-Vehicle / 42-Fishing / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / 471-Wing / 856-Airbus A320 family / \n",
"8-Football / 21-Stadium / \n",
"1-Vehicle / 4-Car / 406-Chevrolet / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 74-Truck / 124-Tractor / 252-Tractor pulling / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 70-Driving / 169-Rallying / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 161-Train station / 224-Rapid transit / \n",
"1312-Dynasty Warriors / 3721-Dynasty Warriors 7 / 3813-Dynasty Warriors: Gundam / 3897-Koei / \n",
"131-Computer / \n",
"1-Vehicle / 74-Truck / \n",
"0-Games / 6-Animation / 15-Cartoon / \n",
"20-Trailer / 3792-Dwarf / \n",
"1854-Centella / \n",
"6-Animation / 126-The Walt Disney Company / 144-Amusement park / 576-Walt Disney World / 679-Mickey Mouse / 1234-Mouse / 1327-Minnie Mouse / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 2-Video game / 216-Call of Duty: Modern Warfare 3 / \n",
"3-Concert / \n",
"330-Money / 731-Coin / 2390-Dollar / 2937-Banknote / \n",
"9-Music video / 28-Fashion / 153-Dress / 483-Costume / \n",
"75-Wrestling / \n",
"3-Concert / \n",
"627-Clock / 3312-Seiko / 3750-It's a Small World / \n",
"75-Wrestling / \n",
"0-Games / 1-Vehicle / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 393-Unmanned aerial vehicle / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"0-Games / 8-Football / 79-American football / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 77-Arena / 81-Athlete / \n",
"12-Food / 242-Paper / \n",
"93-Comedy / \n",
"93-Comedy / \n",
"12-Food / 26-Cooking / 58-Cuisine / 232-Cake / \n",
"219-Accordion / 1994-Garmon / \n",
"28-Fashion / 894-Jacket / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 172-Motocross / \n",
"102-Building / 113-House / 339-Architecture / \n",
"0-Games / 6-Animation / 15-Cartoon / 25-Toy / 126-The Walt Disney Company / \n",
"1-Vehicle / 4-Car / 74-Truck / 158-Tire / 167-Four-wheel drive / 375-Pickup truck / 515-Dodge / \n",
"307-Hand / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 380-Assassin's Creed / \n",
"49-School / 444-Family / \n",
"8-Football / \n",
"40-Road / 70-Driving / \n",
"5-Dance / 16-Performance art / 95-Talent show / \n",
"59-Winter sport / 84-Snow / 177-Skiing / 305-Ski / 789-Alpine skiing / 1830-Downhill / \n",
"61-Art / 148-Painting / 641-Mask / \n",
"1-Vehicle / 4-Car / 18-Outdoor recreation / 22-Nature / 25-Toy / 74-Truck / 92-Radio-controlled model / 147-River / 202-Radio-controlled car / 787-Monster truck / \n",
"20-Trailer / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 70-Driving / 169-Rallying / 482-Asphalt / 1634-Ford Fiesta / \n",
"36-Piano / \n",
"1-Vehicle / \n",
"219-Accordion / 655-Diatonic button accordion / \n",
"0-Games / 8-Football / 21-Stadium / 81-Athlete / \n",
"0-Games / 8-Football / \n",
"10-Animal / 48-Pet / 132-Bird / \n",
"0-Games / 326-Metin2 / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 51-Strategy video game / 127-Xbox 360 / 194-Call of Duty: Modern Warfare 2 / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 96-Soldier / 464-First-person Shooter / 1543-Crysis / \n",
"0-Games / 60-Basketball / 255-Slam dunk / \n",
"0-Games / 35-Minecraft / \n",
"1-Vehicle / 4-Car / 158-Tire / \n",
"7-Musician / 13-Musical ensemble / 39-Drums / 44-Drums / 146-Snare drum / \n",
"5-Dance / 59-Winter sport / 112-Ice skating / 262-Ice / 316-Figure skating / 833-Ice skate / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 77-Arena / 81-Athlete / \n",
"23-Mobile phone / 29-Smartphone / 242-Paper / 370-Origami / 631-Cube / \n",
"47-Personal computer / 131-Computer / 309-Computer hardware / 602-Central processing unit / 957-Desktop computer / 1045-Computer case / \n",
"985-Strawberry / \n",
"102-Building / 113-House / 339-Architecture / \n",
"89-Comics / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 20-Trailer / 35-Minecraft / 394-Map / \n",
"8-Football / 595-Champion / \n",
"31-Disc jockey / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"3-Concert / 7-Musician / 24-String instrument / \n",
"45-Cosmetics / 56-Hair / 871-Hair coloring / 1269-Human hair color / 3540-Violet / \n",
"6-Animation / 15-Cartoon / \n",
"8-Football / 54-Highlight film / 73-Ball / 595-Champion / \n",
"14-Guitar / 24-String instrument / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / 265-Electronic keyboard / 592-Synthesia / \n",
"6-Animation / 15-Cartoon / 60-Basketball / \n",
"113-House / \n",
"21-Stadium / 463-Cricket / 1495-Twenty20 / \n",
"37-Gadget / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 70-Driving / 1324-Roadster / 1907-Ferrari F430 / 4020-Ferrari 360 / \n",
"134-Call of Duty: Black Ops / 163-Running / \n",
"10-Animal / 25-Toy / 97-Drawing / \n",
"12-Food / 1044-Coca-Cola / \n",
"98-Festival / \n",
"1-Vehicle / 4-Car / 94-Dashcam / 155-Camera / \n",
"783-Zee TV / \n",
"12-Food / 136-Gardening / 227-Water / 259-Fire / \n",
"121-Photography / \n",
"0-Games / 240-Arcade game / 1802-Jubeat / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 76-Ball / 77-Arena / 81-Athlete / \n",
"10-Animal / 42-Fishing / 91-Fish / 268-Recreational fishing / 510-Fishing lure / \n",
"0-Games / 159-World of Warcraft / 171-Warcraft / \n",
"7-Musician / 14-Guitar / 24-String instrument / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / \n",
"3-Concert / 16-Performance art / \n",
"345-Loudspeaker / 484-Subwoofer / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 172-Motocross / \n",
"1035-Bank / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 219-Accordion / \n",
"93-Comedy / \n",
"12-Food / 26-Cooking / 32-Recipe / 786-Juice / 1329-Blender / \n",
"3-Concert / \n",
"31-Disc jockey / \n",
"2-Video game / 90-Sports game / 180-FIFA 15 / 480-FIFA 13 / \n",
"93-Comedy / 341-Sketch comedy / \n",
"12-Food / 86-Plant / 156-Agriculture / 1158-Wheat / 2920-Food grain / 3260-Barley / \n",
"20-Trailer / \n",
"4545-Arwen / \n",
"0-Games / 25-Toy / 455-Unidentified flying object / 1498-Key / 1668-Claw crane / \n",
"256-Tennis / 974-Racket / 1337-Serve / \n",
"122-Weight training / 214-Muscle / 1250-Biceps / 2045-Biceps curl / \n",
"20-Trailer / 325-Vampire / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 179-Kitchen / 215-Eating / 295-Chocolate / 301-Meal / 1024-Culinary art / \n",
"3-Concert / 296-Saxophone / \n",
"174-Farm / 348-Fruit / 366-Dragon / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 910-Dragon Quest / 4172-Final Fantasy III / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 3263-Shadow of the Colossus / \n",
"452-Watch / \n",
"205-Newscaster / 273-News program / \n",
"66-Bollywood / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 109-Cooking show / 236-Baking / 391-Chicken meat / 1081-Lemon / 2309-Low-carbohydrate diet / \n",
"66-Bollywood / \n",
"3-Concert / \n",
"28-Fashion / 56-Hair / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 6-Animation / 19-PC game / 1077-Lightning McQueen / \n",
"1-Vehicle / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 197-Trail / 220-Mountain bike / 284-Mountain biking / \n",
"3-Concert / 5-Dance / 7-Musician / 16-Performance art / \n",
"248-Hotel / 288-Underwater / \n",
"2-Video game / 189-iPad / 1150-iPad 2 / \n",
"5-Dance / 85-Combat / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 1144-Castlevania / \n",
"0-Games / 2-Video game / 90-Sports game / 180-FIFA 15 / 480-FIFA 13 / \n",
"222-Surfing / 225-Lake / 233-Ocean / \n",
"6-Animation / \n",
"20-Trailer / \n",
"12-Food / 88-Machine / 208-Wood / 812-Plastic / \n",
"42-Fishing / \n",
"18-Outdoor recreation / 84-Snow / 120-Winter / 262-Ice / 2058-Snowman / \n",
"12-Food / 85-Combat / 259-Fire / 620-Barbecue / \n",
"28-Fashion / 56-Hair / 106-Hairstyle / \n",
"5-Dance / 25-Toy / 130-Gymnastics / \n",
"5-Dance / 117-Boxing / \n",
"9-Music video / \n",
"31-Disc jockey / \n",
"28-Fashion / 45-Cosmetics / 1811-Forever 21 / 2963-Converse / \n",
"6-Animation / 15-Cartoon / 25-Toy / 290-Pony / 343-My Little Pony / \n",
"0-Games / 2-Video game / 37-Gadget / 55-Video game console / 292-Handheld game console / 317-PlayStation / 362-PlayStation Portable / 387-Spider-Man / 682-PlayStation Vita / 3561-The Amazing Spider-Man / \n",
"0-Games / 2-Video game / 15-Cartoon / 241-Fighting game / 486-M.U.G.E.N / \n",
"97-Drawing / 502-Sky / 559-Night / \n",
"7-Musician / 38-Orchestra / 150-Violin / \n",
"7-Musician / 38-Orchestra / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / 318-Church / 870-Chapel / \n",
"164-Beach / 222-Surfing / 233-Ocean / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / 1127-Drum stick / \n",
"10-Animal / 48-Pet / 132-Bird / \n",
"28-Fashion / \n",
"2-Video game / 55-Video game console / 1083-Role-playing game / 2142-Sega Saturn / 2806-SaGa / \n",
"0-Games / 1-Vehicle / 2-Video game / 27-Motorcycle / 34-Action-adventure game / 47-Personal computer / 201-Grand Theft Auto: San Andreas / 858-Carl Johnson / \n",
"1-Vehicle / 25-Toy / 62-Train / 64-Transport / 119-Locomotive / 152-Track / 428-Rail transport modelling / \n",
"10-Animal / 14-Guitar / 61-Art / 981-Sculpture / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"219-Accordion / 655-Diatonic button accordion / \n",
"75-Wrestling / \n",
"0-Games / 2-Video game / 241-Fighting game / \n",
"0-Games / 159-World of Warcraft / 171-Warcraft / \n",
"7-Musician / 14-Guitar / 100-Electric guitar / \n",
"8-Football / 358-Rugby football / \n",
"3151-Kermit the Frog / \n",
"6-Animation / 15-Cartoon / 809-Logo / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 59-Winter sport / 84-Snow / 120-Winter / 610-Snowmobile / \n",
"49-School / 209-University / \n",
"574-Elevator / 691-Cargo / 1746-Otis Elevator Company / \n",
"1-Vehicle / 4-Car / 41-Sports car / 154-Wheel / 158-Tire / 2868-Nissan 240SX / \n",
"31-Disc jockey / \n",
"631-Cube / 740-Rubik's Cube / \n",
"10-Animal / 18-Outdoor recreation / 217-Hunting / \n",
"3-Concert / 93-Comedy / \n",
"2147-Fiber / 2958-Electrical connector / \n",
"10-Animal / 80-Horse / \n",
"6-Animation / \n",
"3-Concert / 14-Guitar / 16-Performance art / 39-Drums / \n",
"222-Surfing / 672-Bodyboarding / 815-Surfboard / \n",
"5-Dance / \n",
"12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / 210-Cookware and bakeware / 239-Roasting / 1284-Frying / 2483-Deep frying / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 199-Wii / \n",
"2-Video game / \n",
"98-Festival / 365-Fireworks / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 96-Soldier / 111-PlayStation 3 / 194-Call of Duty: Modern Warfare 2 / 216-Call of Duty: Modern Warfare 3 / \n",
"45-Cosmetics / 385-Face / \n",
"0-Games / 2-Video game / 900-MacBook / 1657-MacBook Air / 1928-BioShock / 2645-BioShock Infinite / \n",
"93-Comedy / \n",
"1-Vehicle / 4-Car / 349-Chevrolet / 406-Chevrolet / 3793-Chevrolet Chevy II / Nova / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"1119-Sensor / \n",
"7-Musician / 14-Guitar / 24-String instrument / \n",
"93-Comedy / \n",
"20-Trailer / \n",
"176-PlayStation 4 / \n",
"31-Disc jockey / \n",
"9-Music video / 78-Wedding / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / \n",
"10-Animal / 48-Pet / 166-Cat / 346-Kitten / \n",
"12-Food / 18-Outdoor recreation / 22-Nature / 124-Tractor / 156-Agriculture / 174-Farm / 617-Maize / \n",
"3-Concert / \n",
"502-Sky / \n",
"12-Food / 88-Machine / \n",
"22-Nature / 213-Forest / 423-Coast / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 70-Driving / \n",
"1-Vehicle / 62-Train / 103-Rail transport / 119-Locomotive / 143-Railroad car / 3348-BVE Trainsim / \n",
"85-Combat / 117-Boxing / 1016-Glove / \n",
"6-Animation / 123-Naruto / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / 2656-Guitar Center / \n",
"8-Football / \n",
"88-Machine / 536-Glass / 619-Manufacturing / \n",
"20-Trailer / \n",
"1-Vehicle / 124-Tractor / 136-Gardening / 249-Heavy equipment / 363-Soil / 2138-Compost / \n",
"6-Animation / 15-Cartoon / 2047-Popeye / \n",
"1-Vehicle / 4-Car / 70-Driving / 94-Dashcam / 299-Volkswagen Passenger Cars / \n",
"0-Games / 211-RuneScape / \n",
"9-Music video / \n",
"10-Animal / 48-Pet / 132-Bird / 1767-Cockatiel / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / \n",
"5-Dance / \n",
"61-Art / 97-Drawing / 242-Paper / 768-Diamond / \n",
"10-Animal / 1160-Giant panda / \n",
"8-Football / \n",
"0-Games / 1-Vehicle / 50-Aircraft / \n",
"0-Games / \n",
"130-Gymnastics / 920-Chair / \n",
"5-Dance / 16-Performance art / \n",
"6-Animation / 15-Cartoon / 2408-Lucky Star / \n",
"3-Concert / \n",
"3-Concert / 13-Musical ensemble / \n",
"0-Games / 6-Animation / 10-Animal / 15-Cartoon / 55-Video game console / \n",
"3-Concert / \n",
"0-Games / 256-Tennis / \n",
"56-Hair / 340-Afro-textured hair / 405-Braid / 919-Hair conditioner / \n",
"0-Games / 2-Video game / 60-Basketball / 90-Sports game / \n",
"246-Furniture / 307-Hand / 462-Table / 1057-Embroidery / 3071-Towel / \n",
"2-Video game / 25-Toy / 43-Call of Duty / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 541-Black-and-white / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 70-Driving / 135-Drifting / 1583-Toyota Corolla / 4584-Street Legal Racing: Redline / \n",
"125-Television / 3934-Radiator / \n",
"22-Nature / 164-Beach / 233-Ocean / 275-Diving / 423-Coast / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"5-Dance / 1623-Shaolin Kung Fu / \n",
"196-Eye / \n",
"5-Dance / 78-Wedding / 153-Dress / 190-Bride / 421-Gown / \n",
"117-Boxing / \n",
"88-Machine / 1226-Metalworking / 3073-Log splitter / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 36-Piano / 44-Drums / 150-Violin / \n",
"97-Drawing / 408-Sketch / 528-Pencil / 3491-Stained glass / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"9-Music video / 110-Album / \n",
"130-Gymnastics / 1835-Majorette / \n",
"1-Vehicle / 4-Car / \n",
"0-Games / 2-Video game / 43-Call of Duty / 216-Call of Duty: Modern Warfare 3 / \n",
"3-Concert / 46-Choir / 3707-Yale University / \n",
"3-Concert / 126-The Walt Disney Company / 144-Amusement park / 365-Fireworks / 576-Walt Disney World / 1463-Magic Kingdom / \n",
"16-Performance art / 78-Wedding / 115-Ballet / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 22-Nature / 40-Road / 94-Dashcam / 800-Mountain pass / \n",
"0-Games / 2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 189-iPad / 272-iPod / 331-iPod touch / \n",
"616-The Bible / \n",
"6-Animation / 15-Cartoon / 1674-Ouran High School Host Club / \n",
"56-Hair / 919-Hair conditioner / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 77-Arena / 81-Athlete / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / \n",
"5-Dance / 16-Performance art / 95-Talent show / 115-Ballet / \n",
"5-Dance / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / 361-Flamenco / \n",
"912-Silver / \n",
"0-Games / 49-School / 60-Basketball / 142-Hockey / \n",
"222-Surfing / 233-Ocean / 672-Bodyboarding / 815-Surfboard / \n",
"23-Mobile phone / 29-Smartphone / 178-Samsung Galaxy / 504-Web page / 1287-Samsung Galaxy Tab series / 1390-Samsung Galaxy S II / \n",
"986-Suit / \n",
"1-Vehicle / 9-Music video / 87-Boat / 579-Yacht / \n",
"15-Cartoon / 4525-Hi-Rez Studios / \n",
"10-Animal / 80-Horse / 247-Stallion / 438-Mare / 1871-Foal / 2018-Arabian horse / 2553-Colt / \n",
"9-Music video / \n",
"3-Concert / 5-Dance / 68-Lighting / 144-Amusement park / 366-Dragon / 559-Night / \n",
"8-Football / 595-Champion / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / 44-Drums / 56-Hair / \n",
"20-Trailer / 387-Spider-Man / \n",
"67-Cycling / 215-Eating / 3101-Indoor cycling / \n",
"1862-Corel / \n",
"6-Animation / 25-Toy / 200-Doll / \n",
"79-American football / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 184-Vegetable / 944-Hamburger / 1022-Chili pepper / 1332-Onion / 1355-Vegetarian food / 1381-Garlic / 3100-Chickpea / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 38-Orchestra / \n",
"1-Vehicle / 87-Boat / 308-Ship / 1107-United States Navy / \n",
"1-Vehicle / 27-Motorcycle / 40-Road / 283-Rock / 1358-Canyon / 1808-National park / \n",
"2852-Burger King / \n",
"3051-Sarah Walker / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 210-Cookware and bakeware / 893-Fish as food / 3379-Prawn / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 378-Drag racing / \n",
"6-Animation / 15-Cartoon / 1563-Stick figure / \n",
"22-Nature / 86-Plant / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 89-Comics / 185-Dragon Ball / 1462-Dragon Ball XenoVerse / \n",
"1-Vehicle / 4-Car / 124-Tractor / \n",
"2429-India TV / \n",
"0-Games / 2-Video game / 6-Animation / 111-PlayStation 3 / 547-Bleach / \n",
"3-Concert / 16-Performance art / \n",
"208-Wood / 246-Furniture / \n",
"444-Family / \n",
"0-Games / 582-Candy / 1112-Candy Crush Saga / \n",
"3-Concert / \n",
"5-Dance / 49-School / 191-Cheerleading / \n",
"56-Hair / 106-Hairstyle / \n",
"12-Food / 232-Cake / 1102-Cake decorating / 1365-Fondant icing / \n",
"93-Comedy / 341-Sketch comedy / 364-Sitcom / \n",
"1-Vehicle / 50-Aircraft / 188-Airline / \n",
"10-Animal / \n",
"5-Dance / 49-School / \n",
"31-Disc jockey / 49-School / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"131-Computer / 141-Microsoft Windows / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"25-Toy / 1105-Fingerboard / \n",
"9-Music video / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"1465-FOX / \n",
"0-Games / 2-Video game / 237-Final Fantasy / 4661-Final Fantasy II / \n",
"85-Combat / 117-Boxing / 399-Kickboxing / 699-Kick / \n",
"47-Personal computer / 131-Computer / 228-Laptop / 513-Computer keyboard / 811-Mouse / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 30-Drummer / 39-Drums / 44-Drums / 1044-Coca-Cola / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 24-String instrument / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 150-Violin / \n",
"710-Handball / \n",
"0-Games / 257-Counter-Strike / \n",
"1-Vehicle / 4-Car / 167-Four-wheel drive / 234-Ford / 2899-Ford Explorer / \n",
"110-Album / 2681-Marshmallow / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"1226-Metalworking / 2035-Forge / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / \n",
"5-Dance / 130-Gymnastics / \n",
"356-Star / 654-Sun / 712-Outer space / 774-Galaxy / 2938-Nebula / \n",
"12-Food / 367-Drink / 612-Beer / \n",
"3-Concert / 1684-Hillsong Church / \n",
"130-Gymnastics / 1255-Ribbon / \n",
"2884-Unturned / \n",
"219-Accordion / 655-Diatonic button accordion / \n",
"61-Art / 415-Sewing / 476-Textile / 1398-Triangle / 1479-Quilt / 2373-Square / 2966-Patchwork / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 40-Road / 57-Race track / 278-GoPro / 456-Go-kart / \n",
"22-Nature / \n",
"14-Guitar / \n",
"12-Food / 52-Dish / 278-GoPro / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 1009-iPhone 3G / \n",
"1-Vehicle / 4-Car / 154-Wheel / \n",
"1-Vehicle / 4-Car / 270-Sedan / 449-Honda / 552-Honda Civic / \n",
"3-Concert / \n",
"12-Food / 1162-Dietary supplement / \n",
"13-Musical ensemble / 38-Orchestra / 98-Festival / 253-Carnival / \n",
"5-Dance / 115-Ballet / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 114-Engine / 3004-Softail / \n",
"1-Vehicle / 4-Car / 1391-Kia / 1415-Kia Motors / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 127-Xbox 360 / 1419-GameSpot / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 3977-Lost Planet / \n",
"144-Amusement park / 321-Roller coaster / 334-Amusement ride / 1059-Movie theater / 2786-3D television / \n",
"0-Games / 2-Video game / 121-Photography / 125-Television / 1348-Panasonic Corporation / \n",
"6-Animation / 15-Cartoon / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 39-Drums / \n",
"0-Games / 2-Video game / 75-Wrestling / 1582-WWE 2K14 / \n",
"8-Football / 67-Cycling / 69-Bicycle / 570-Road bicycle racing / \n",
"8-Football / \n",
"5-Dance / 115-Ballet / \n",
"7-Musician / 14-Guitar / 266-Helicopter / \n",
"694-Tai chi / \n",
"42-Fishing / \n",
"5-Dance / \n",
"5-Dance / 66-Bollywood / \n",
"10-Animal / 763-Snake / \n",
"12-Food / 3544-Rubber stamp / \n",
"0-Games / 116-Pokémon / 186-Pokémon / 3347-Misty / \n",
"0-Games / 2-Video game / 6-Animation / 33-Weapon / 229-Halo / \n",
"5-Dance / \n",
"9-Music video / \n",
"3-Concert / \n",
"3-Concert / 30-Drummer / \n",
"9-Music video / \n",
"10-Animal / 22-Nature / 91-Fish / 2829-Dictionary / \n",
"0-Games / 8-Football / \n",
"0-Games / 242-Paper / 370-Origami / \n",
"0-Games / 725-Human / 1698-Darts / \n",
"3-Concert / \n",
"5-Dance / 130-Gymnastics / 1474-Acrobatic gymnastics / \n",
"66-Bollywood / \n",
"1-Vehicle / 62-Train / 64-Transport / 94-Dashcam / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / 224-Rapid transit / \n",
"1-Vehicle / 11-Motorsport / 124-Tractor / 156-Agriculture / 174-Farm / 252-Tractor pulling / 363-Soil / 459-Plough / \n",
"3-Concert / 7-Musician / 16-Performance art / 30-Drummer / \n",
"9-Music video / \n",
"7-Musician / 13-Musical ensemble / 14-Guitar / 44-Drums / \n",
"28-Fashion / 56-Hair / 106-Hairstyle / 405-Braid / \n",
"0-Games / 117-Boxing / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 36-Piano / 230-Brass instrument / 427-Trumpet / \n",
"10-Animal / 48-Pet / 132-Bird / 598-Parrot / 1767-Cockatiel / 2024-Cockatoo / \n",
"1-Vehicle / 33-Weapon / 271-Tank / \n",
"33-Weapon / \n",
"25-Toy / 200-Doll / 1828-Campervan / 1842-Dollhouse / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / 49-School / \n",
"3-Concert / \n",
"144-Amusement park / 321-Roller coaster / 334-Amusement ride / \n",
"6-Animation / 15-Cartoon / 877-Hatsune Miku: Project DIVA / 1648-Hatsune Miku: Project DIVA F / 2682-Hatsune Miku: Project DIVA F 2nd / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 89-Comics / 123-Naruto / 285-Sasuke Uchiha / 591-Naruto: Ultimate Ninja / 2299-Naruto Shippuden: Ultimate Ninja Storm 2 / \n",
"46-Choir / \n",
"9-Music video / \n",
"28-Fashion / 45-Cosmetics / 429-Website / \n",
"3-Concert / 7-Musician / 36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / 878-Composer / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 975-Dune / \n",
"122-Weight training / 162-Medicine / 214-Muscle / 1162-Dietary supplement / \n",
"18-Outdoor recreation / 42-Fishing / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / 1031-Fisheye lens / \n",
"1-Vehicle / 4-Car / 87-Boat / 246-Furniture / 579-Yacht / \n",
"1-Vehicle / 11-Motorsport / 22-Nature / 27-Motorcycle / 197-Trail / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"6-Animation / 15-Cartoon / 99-Christmas / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / \n",
"12-Food / 26-Cooking / 179-Kitchen / 215-Eating / \n",
"3-Concert / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / \n",
"0-Games / 60-Basketball / 168-Basketball moves / 4466-Korfball / \n",
"3-Concert / 98-Festival / 594-Music festival / \n",
"56-Hair / 106-Hairstyle / \n",
"0-Games / 2-Video game / 229-Halo / 470-Halo 3 / \n",
"6-Animation / 15-Cartoon / \n",
"6-Animation / 15-Cartoon / 20-Trailer / 294-Batman / 2069-Flash / 3473-Nightwing / \n",
"45-Cosmetics / 56-Hair / 235-Lipstick / 282-Skin / 385-Face / \n",
"3-Concert / 31-Disc jockey / 68-Lighting / 133-Nightclub / 263-Light / \n",
"2-Video game / 1985-The Last of Us / \n",
"843-The Doctor / \n",
"0-Games / 8-Football / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / \n",
"6-Animation / 15-Cartoon / 20-Trailer / 116-Pokémon / 183-Manga / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 8-Football / 79-American football / \n",
"23-Mobile phone / 101-Telephone / \n",
"5-Dance / \n",
"6-Animation / 330-Money / 402-Jewellery / \n",
"6-Animation / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / 1197-Hedgehog / 1299-Shadow the Hedgehog / 3321-Shadow the Hedgehog / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / \n",
"78-Wedding / 136-Gardening / \n",
"5-Dance / 16-Performance art / 95-Talent show / \n",
"1-Vehicle / 50-Aircraft / 778-Gliding / 1263-Glider / 1317-Glider / \n",
"162-Medicine / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / 2345-Kawasaki Ninja 250R / \n",
"1-Vehicle / 4-Car / 1242-Car wash / \n",
"0-Games / 2-Video game / 19-PC game / 257-Counter-Strike / 702-Counter-Strike: Source / \n",
"0-Games / 2-Video game / 264-Star Wars / \n",
"125-Television / \n",
"4214-Spike / \n",
"59-Winter sport / 84-Snow / 120-Winter / 177-Skiing / 305-Ski / \n",
"61-Art / 148-Painting / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / 1512-Sonic Generations / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 85-Combat / 111-PlayStation 3 / 134-Call of Duty: Black Ops / 677-Game controller / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 56-Hair / 194-Call of Duty: Modern Warfare 2 / \n",
"22-Nature / 86-Plant / 136-Gardening / 279-Garden / \n",
"0-Games / 8-Football / 90-Sports game / \n",
"25-Toy / 200-Doll / 412-Monster / 503-Monster High / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / \n",
"0-Games / 2-Video game / 19-PC game / 34-Action-adventure game / 118-Grand Theft Auto V / 1069-Easter egg / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / \n",
"75-Wrestling / 85-Combat / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 203-Marching band / 230-Brass instrument / \n",
"0-Games / 2-Video game / 33-Weapon / 134-Call of Duty: Black Ops / \n",
"1-Vehicle / 25-Toy / 33-Weapon / 140-LEGO / 271-Tank / \n",
"3-Concert / 46-Choir / 318-Church / \n",
"3-Concert / 545-Quartet (ensemble) / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 197-Trail / 220-Mountain bike / 284-Mountain biking / \n",
"10-Animal / 86-Plant / 91-Fish / 136-Gardening / 258-Aquarium / 1258-Goldfish / \n",
"8-Football / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / 81-Athlete / \n",
"6-Animation / 15-Cartoon / 379-Touhou Project / 2170-The Embodiment of Scarlet Devil / \n",
"10-Animal / 48-Pet / 71-Dog / 2059-Doberman Pinscher / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 17-Racing / 27-Motorcycle / 40-Road / 212-Highway / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"9-Music video / \n",
"463-Cricket / 1476-Bowling (cricket) / \n",
"9-Music video / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 247-Stallion / \n",
"28-Fashion / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / \n",
"59-Winter sport / 177-Skiing / 373-Clash of Clans / 1764-Ski jumping / \n",
"3-Concert / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 1614-Honda Accord / \n",
"283-Rock / \n",
"0-Games / 2-Video game / 55-Video game console / 292-Handheld game console / 320-Super Smash Bros. / \n",
"5-Dance / 306-High school / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / 860-Christian Church / \n",
"44-Drums / 246-Furniture / 1825-Cajón / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 247-Stallion / 289-Dressage / 351-Jumping / \n",
"12-Food / 26-Cooking / 179-Kitchen / \n",
"47-Personal computer / 131-Computer / 497-Macintosh / 504-Web page / \n",
"39-Drums / 44-Drums / 146-Snare drum / \n",
"1-Vehicle / 2-Video game / 50-Aircraft / \n",
"5-Dance / 66-Bollywood / 4017-Television film / \n",
"5-Dance / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 73-Ball / 76-Ball / 77-Arena / \n",
"0-Games / 2-Video game / 596-Monster Hunter / 1935-Monster Hunter Tri / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 42-Fishing / 91-Fish / 217-Hunting / 275-Diving / 288-Underwater / 381-Underwater diving / 407-Scuba diving / \n",
"6-Animation / 15-Cartoon / 577-Penguin / \n",
"8-Football / \n",
"325-Vampire / \n",
"897-Avengers / 1469-X-Men / \n",
"1-Vehicle / 4-Car / 124-Tractor / 156-Agriculture / 174-Farm / 363-Soil / 459-Plough / 906-Sowing / \n",
"223-Weather / \n",
"10-Animal / 80-Horse / 247-Stallion / 1434-Saddle / \n",
"61-Art / 457-Knitting / 546-Stitch / \n",
"5-Dance / 76-Ball / 2389-Exercise ball / \n",
"141-Microsoft Windows / \n",
"79-American football / 303-Paint / 522-Helmet / \n",
"6-Animation / \n",
"2867-Wasp / \n",
"8-Football / \n",
"1-Vehicle / 4-Car / 40-Road / 259-Fire / 371-Firefighter / 1633-Fire station / \n",
"5-Dance / 16-Performance art / 98-Festival / \n",
"2196-Boiler / \n",
"0-Games / 35-Minecraft / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 81-Athlete / \n",
"6-Animation / 185-Dragon Ball / 200-Doll / 242-Paper / \n",
"23-Mobile phone / 29-Smartphone / 204-Gym / 2488-Fitness Center / \n",
"7-Musician / 14-Guitar / \n",
"0-Games / 1047-Easter egg / \n",
"0-Games / 2-Video game / 85-Combat / 803-DC Comics / 1515-DC Universe Online / 3068-Green Arrow / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / \n",
"6-Animation / 15-Cartoon / \n",
"5-Dance / 16-Performance art / \n",
"30-Drummer / 39-Drums / 44-Drums / 123-Naruto / 128-Cymbal / 146-Snare drum / \n",
"20-Trailer / \n",
"0-Games / 25-Toy / 76-Ball / 323-Toddler / \n",
"46-Choir / \n",
"5-Dance / \n",
"18-Outdoor recreation / 22-Nature / 144-Amusement park / 321-Roller coaster / 334-Amusement ride / 3395-Asterix / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / 158-Tire / 207-Exhaust system / 2617-Kawasaki Ninja ZX-10R / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 19-PC game / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"0-Games / 211-RuneScape / 2685-Giant / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 40-Road / \n",
"48-Pet / 159-World of Warcraft / 171-Warcraft / \n",
"0-Games / 2-Video game / 43-Call of Duty / 118-Grand Theft Auto V / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / 1543-Crysis / \n",
"5-Dance / \n",
"28-Fashion / 56-Hair / 106-Hairstyle / 919-Hair conditioner / 1359-Hair iron / 3584-Hair roller / \n",
"1-Vehicle / 18-Outdoor recreation / 59-Winter sport / 84-Snow / 120-Winter / 610-Snowmobile / \n",
"10-Animal / 48-Pet / 71-Dog / 84-Snow / \n",
"9-Music video / \n",
"170-Home improvement / 246-Furniture / 830-Bed / \n",
"121-Photography / 155-Camera / 721-Digital SLR / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / \n",
"0-Games / 8-Football / \n",
"0-Games / 2-Video game / 1144-Castlevania / \n",
"89-Comics / \n",
"3540-Violet / \n",
"61-Art / 89-Comics / 294-Batman / 435-Action figure / 737-Joker / \n",
"66-Bollywood / \n",
"1-Vehicle / 11-Motorsport / 18-Outdoor recreation / 27-Motorcycle / \n",
"170-Home improvement / 303-Paint / 353-Wall / 1179-Brick / \n",
"85-Combat / 1832-Hand-to-hand combat / \n",
"2620-Federal Reserve System / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 345-Loudspeaker / \n",
"1-Vehicle / 4-Car / 509-Classic car / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 40-Road / 74-Truck / 118-Grand Theft Auto V / 886-James Bond / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"9-Music video / \n",
"0-Games / 12-Food / 88-Machine / 822-Crane / 1314-Garage / 1668-Claw crane / \n",
"0-Games / 2-Video game / 240-Arcade game / 1467-Pump It Up / 2651-Pump It Up Fiesta / \n",
"0-Games / 2-Video game / 20-Trailer / 35-Minecraft / \n",
"1-Vehicle / 715-Farming Simulator / \n",
"3-Concert / 5-Dance / 13-Musical ensemble / \n",
"6-Animation / 15-Cartoon / 1338-Devil / \n",
"1-Vehicle / 27-Motorcycle / 105-Motorcycling / 114-Engine / 806-Chopper / \n",
"6-Animation / 1192-Hello Kitty / \n",
"5-Dance / 703-Roller skating / \n",
"0-Games / 2-Video game / 192-Battlefield / 374-Battlefield 3 / \n",
"376-Flute / 3344-Tin whistle / \n",
"465-Microphone / \n",
"2-Video game / 6-Animation / 15-Cartoon / 111-PlayStation 3 / 123-Naruto / 413-PlayStation 2 / 591-Naruto: Ultimate Ninja / \n",
"523-Cattle / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 709-Bull / 824-Terrier / 2388-Bull Terrier / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / 422-Call of Duty: Ghosts / \n",
"6-Animation / 15-Cartoon / \n",
"20-Trailer / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"102-Building / 113-House / 339-Architecture / 416-Apartment / 939-Condominium / \n",
"434-Ninja / 865-Turtle / 951-Teenage Mutant Ninja Turtles / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"7-Musician / 46-Choir / 3463-Nun / \n",
"31-Disc jockey / \n",
"3-Concert / \n",
"31-Disc jockey / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 79-American football / 81-Athlete / \n",
"6-Animation / \n",
"9-Music video / \n",
"170-Home improvement / 492-Floor / 934-Flooring / 3359-Laminate flooring / \n",
"125-Television / 209-University / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 3058-BMW 7 Series / \n",
"0-Games / 2-Video game / 192-Battlefield / 374-Battlefield 3 / 383-Battlefield 4 / \n",
"1-Vehicle / 11-Motorsport / \n",
"60-Basketball / \n",
"6-Animation / 15-Cartoon / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 169-Rallying / \n",
"10-Animal / 136-Gardening / 458-Gold / 521-Metal / 731-Coin / 959-Metal detector / \n",
"121-Photography / 155-Camera / 721-Digital SLR / 1119-Sensor / \n",
"1-Vehicle / 4-Car / 270-Sedan / 827-Pump / 2509-Mazda3 / \n",
"122-Weight training / 204-Gym / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 2594-Mount & Blade: Warband / 2622-Mount & Blade / \n",
"45-Cosmetics / 173-Eye shadow / \n",
"5-Dance / \n",
"1-Vehicle / 4-Car / 88-Machine / 114-Engine / 741-Mower / \n",
"66-Bollywood / \n",
"10-Animal / 48-Pet / 71-Dog / 293-Slide show / \n",
"156-Agriculture / 174-Farm / 328-Simulation video game / 715-Farming Simulator / \n",
"36-Piano / 53-Keyboard / 104-Pianist / \n",
"0-Games / 2-Video game / 269-Comic book / 489-Board game / \n",
"28-Fashion / 93-Comedy / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 105-Motorcycling / 172-Motocross / 491-Enduro / \n",
"260-Prayer / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 344-Coupé / 569-Lamborghini / 1763-Lotus Cars / 2400-Asphalt / 4481-Pagani Huayra / \n",
"93-Comedy / \n",
"12-Food / 26-Cooking / 32-Recipe / 109-Cooking show / 1174-Smoothie / \n",
"1-Vehicle / 83-Skateboarding / 108-Skateboard / \n",
"0-Games / 2-Video game / 240-Arcade game / 241-Fighting game / 311-Street Fighter / 461-Street Fighter IV / 496-Super Street Fighter IV / 563-Ryu / \n",
"6-Animation / 15-Cartoon / \n",
"28-Fashion / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"6-Animation / 434-Ninja / 951-Teenage Mutant Ninja Turtles / 3508-Leonardo / 3667-Raphael / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 216-Call of Duty: Modern Warfare 3 / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"1-Vehicle / 4-Car / 18-Outdoor recreation / 74-Truck / 137-Tree / 158-Tire / 167-Four-wheel drive / 197-Trail / 198-Off-road vehicle / 234-Ford / 327-Mud / 360-Jeep / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 118-Grand Theft Auto V / 176-PlayStation 4 / \n",
"1-Vehicle / 249-Heavy equipment / 433-Construction / \n",
"3-Concert / 16-Performance art / \n",
"8-Football / \n",
"5-Dance / 259-Fire / 350-Circus / 788-Juggling / \n",
"0-Games / 1-Vehicle / 156-Agriculture / 174-Farm / 715-Farming Simulator / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 30-Drummer / 39-Drums / 44-Drums / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 134-Call of Duty: Black Ops / \n",
"1-Vehicle / 4-Car / 40-Road / 94-Dashcam / 212-Highway / \n",
"0-Games / 34-Action-adventure game / 55-Video game console / 118-Grand Theft Auto V / 127-Xbox 360 / 386-Xbox / \n",
"10-Animal / 12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 91-Fish / 893-Fish as food / \n",
"5-Dance / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"0-Games / 489-Board game / 3863-Temple Run / \n",
"8-Football / \n",
"9-Music video / \n",
"319-Robot / 1885-Robotics / \n",
"410-Teacher / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 101-Telephone / \n",
"10-Animal / 12-Food / 91-Fish / 3103-Green tea / \n",
"1-Vehicle / 4-Car / 299-Volkswagen Passenger Cars / 531-Hatchback / 668-Volkswagen Golf / 2214-Multi-valve / 2286-Grand Tourer Injection / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 302-Call of Duty 4: Modern Warfare / \n",
"46-Choir / 716-Coral / \n",
"67-Cycling / 69-Bicycle / 1067-Lock / 2516-Lock picking / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 6-Animation / 111-PlayStation 3 / 2984-.hack / \n",
"3-Concert / 16-Performance art / \n",
"31-Disc jockey / 322-Mixtape / \n",
"0-Games / 8-Football / 73-Ball / \n",
"131-Computer / 141-Microsoft Windows / \n",
"0-Games / 8-Football / 65-Kick / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 127-Xbox 360 / 2205-Bully / \n",
"310-The Sims / 543-The Sims 2 / \n",
"88-Machine / 400-Woodturning / \n",
"1-Vehicle / 27-Motorcycle / 105-Motorcycling / 207-Exhaust system / \n",
"18-Outdoor recreation / 59-Winter sport / 84-Snow / 177-Skiing / 305-Ski / \n",
"5-Dance / 115-Ballet / \n",
"12-Food / 184-Vegetable / 348-Fruit / \n",
"0-Games / 8-Football / 73-Ball / \n",
"0-Games / 8-Football / \n",
"110-Album / \n",
"3-Concert / 5-Dance / 16-Performance art / 95-Talent show / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 135-Drifting / 231-Supercar / 711-Gran Turismo / 953-Gran Turismo 5 / \n",
"0-Games / 21-Stadium / 54-Highlight film / 79-American football / \n",
"7-Musician / 36-Piano / 53-Keyboard / 104-Pianist / \n",
"365-Fireworks / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"59-Winter sport / 81-Athlete / 112-Ice skating / 142-Hockey / 262-Ice / 312-Ice rink / 833-Ice skate / 1973-Hockey puck / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 447-Sand / 751-Desert / 975-Dune / 2875-Quadracycle / \n",
"6-Animation / 15-Cartoon / 183-Manga / 701-Beyblade / \n",
"9-Music video / 13-Musical ensemble / 14-Guitar / 39-Drums / 44-Drums / \n",
"75-Wrestling / 435-Action figure / 1441-Mattel / \n",
"10-Animal / 227-Water / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / \n",
"0-Games / 287-Human swimming / \n",
"1-Vehicle / 8-Football / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"1107-United States Navy / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"5-Dance / 333-Nike; Inc. / \n",
"23-Mobile phone / 29-Smartphone / 125-Television / \n",
"955-Police officer / \n",
"223-Weather / \n",
"93-Comedy / \n",
"38-Orchestra / \n",
"0-Games / 2-Video game / \n",
"1-Vehicle / 4-Car / \n",
"3-Concert / 5-Dance / 98-Festival / \n",
"22-Nature / 67-Cycling / 69-Bicycle / 179-Kitchen / 197-Trail / 213-Forest / 220-Mountain bike / 284-Mountain biking / \n",
"1-Vehicle / 4-Car / 64-Transport / 74-Truck / 259-Fire / 371-Firefighter / 1633-Fire station / \n",
"1-Vehicle / 18-Outdoor recreation / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 2-Video game / 4468-Breath of Fire / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / \n",
"2-Video game / 1028-Devil May Cry / 2336-Devil May Cry 3: Dante's Awakening / \n",
"28-Fashion / 45-Cosmetics / 56-Hair / 106-Hairstyle / \n",
"47-Personal computer / 145-Tablet computer / 189-iPad / \n",
"2614-Lettuce / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 234-Ford / 406-Chevrolet / 473-Ford Mustang / 509-Classic car / 761-Chevrolet Camaro / 1148-Hot rod / 2061-Chevrolet Chevelle / 2078-Dodge Challenger / \n",
"122-Weight training / 214-Muscle / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"648-Clown / \n",
"0-Games / 6-Animation / 25-Toy / \n",
"0-Games / 60-Basketball / 168-Basketball moves / 255-Slam dunk / \n",
"14-Guitar / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 24-String instrument / 150-Violin / 401-Cello / 450-Viola / 545-Quartet (ensemble) / 639-Fiddle / 1002-String quartet / \n",
"0-Games / 23-Mobile phone / 29-Smartphone / 492-Floor / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 584-Super Mario Bros. / \n",
"298-Television advertisement / 337-Advertising / \n",
"376-Flute / 1326-Recorder / \n",
"12-Food / \n",
"3-Concert / 31-Disc jockey / 98-Festival / \n",
"0-Games / 2-Video game / 60-Basketball / 90-Sports game / 1262-NBA 2K14 / \n",
"0-Games / 54-Highlight film / 60-Basketball / 77-Arena / 81-Athlete / 168-Basketball moves / 255-Slam dunk / \n",
"3-Concert / 30-Drummer / \n",
"53-Keyboard / 1909-Analog synthesizer / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 270-Sedan / 680-Subaru / 1286-Subaru / \n",
"6-Animation / 483-Costume / 596-Monster Hunter / \n",
"0-Games / 2-Video game / 903-Mass Effect / \n",
"1-Vehicle / 27-Motorcycle / 3410-Paper model / \n",
"20-Trailer / \n",
"0-Games / 455-Unidentified flying object / 1668-Claw crane / \n",
"0-Games / 756-Tibia / \n",
"0-Games / 781-Pachinko / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 96-Soldier / 192-Battlefield / 383-Battlefield 4 / \n",
"8-Football / 76-Ball / \n",
"5-Dance / 444-Family / \n",
"1-Vehicle / 4-Car / 259-Fire / \n",
"3-Concert / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 232-Cake / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 81-Athlete / \n",
"163-Running / \n",
"22-Nature / 136-Gardening / 137-Tree / 642-Leaf / 657-Autumn / \n",
"3-Concert / 16-Performance art / \n",
"313-Book / 1640-Library / \n",
"3-Concert / \n",
"1057-Embroidery / 1451-Ribbon / \n",
"5-Dance / 46-Choir / 49-School / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / \n",
"0-Games / 6-Animation / 15-Cartoon / 89-Comics / 185-Dragon Ball / 244-Goku / 418-Gohan / 869-Dragon Ball Z: Budokai Tenkaichi / \n",
"10-Animal / 1591-Leopard / 3055-Gecko / \n",
"356-Star / 419-Earth / 712-Outer space / 1702-Telescope / 2643-Asteroid / 3511-Meteoroid / 3942-Meteor shower / \n",
"75-Wrestling / 85-Combat / 3305-Tag team / 3912-Face / \n",
"412-Monster / \n",
"3-Concert / 7-Musician / \n",
"640-Bottle / 1510-Liquid / 1544-Laboratory / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / \n",
"6-Animation / 15-Cartoon / 1527-Pretty Cure / \n",
"47-Personal computer / 131-Computer / 228-Laptop / \n",
"42-Fishing / 370-Origami / 510-Fishing lure / \n",
"12-Food / 58-Cuisine / \n",
"36-Piano / 110-Album / 585-Lawn / \n",
"0-Games / 2-Video game / 20-Trailer / 55-Video game console / 111-PlayStation 3 / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 55-Video game console / 2249-Driver / \n",
"3-Concert / 16-Performance art / \n",
"0-Games / 127-Xbox 360 / 386-Xbox / \n",
"0-Games / 60-Basketball / 4355-Netball / \n",
"425-City / 1528-Euro / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"0-Games / 2-Video game / 116-Pokémon / 186-Pokémon / 1387-Pokémon Battle Revolution / \n",
"1-Vehicle / 18-Outdoor recreation / 27-Motorcycle / 67-Cycling / 69-Bicycle / 327-Mud / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / 376-Flute / \n",
"0-Games / 2-Video game / 51-Strategy video game / \n",
"25-Toy / 4670-Footbag / \n",
"9-Music video / \n",
"96-Soldier / 205-Newscaster / 444-Family / \n",
"0-Games / 2-Video game / 432-Kingdom Hearts / 665-Kingdom Hearts / 836-Sora / 941-Kingdom Hearts II / 1730-Roxas / \n",
"47-Personal computer / 131-Computer / 228-Laptop / 309-Computer hardware / \n",
"10-Animal / 537-Zoo / 571-Bear / 1160-Giant panda / \n",
"0-Games / 60-Basketball / 75-Wrestling / 595-Champion / \n",
"20-Trailer / \n",
"0-Games / 59-Winter sport / 112-Ice skating / 142-Hockey / 262-Ice / 312-Ice rink / \n",
"365-Fireworks / \n",
"3-Concert / \n",
"3-Concert / \n",
"110-Album / \n",
"1232-Perfect World / 2394-Perfect World / \n",
"12-Food / 2926-Barista / \n",
"5-Dance / \n",
"9-Music video / \n",
"3-Concert / 5-Dance / 16-Performance art / 98-Festival / \n",
"122-Weight training / 532-Barbell / 1250-Biceps / \n",
"1-Vehicle / 4-Car / 158-Tire / 2742-Mini 4WD / \n",
"1-Vehicle / 50-Aircraft / 129-Aviation / 151-Landing / 157-Jet aircraft / 175-Airliner / \n",
"1-Vehicle / 4-Car / 449-Honda / 552-Honda Civic / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"31-Disc jockey / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 84-Snow / 120-Winter / 135-Drifting / 344-Coupé / 683-Renault / 2455-Renault Mégane / \n",
"2093-Coating / \n",
"5-Dance / 396-One Piece / \n",
"6-Animation / \n",
"0-Games / 2-Video game / 637-Five Nights at Freddy's / \n",
"110-Album / \n",
"769-Red Bull / \n",
"3-Concert / \n",
"412-Monster / \n",
"78-Wedding / \n",
"0-Games / 33-Weapon / 229-Halo / 470-Halo 3 / \n",
"31-Disc jockey / \n",
"10-Animal / 80-Horse / 351-Jumping / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 84-Snow / 118-Grand Theft Auto V / 176-PlayStation 4 / \n",
"121-Photography / 512-Printing / 525-Printer / 2634-Inkjet printing / \n",
"12-Food / 26-Cooking / 389-Restaurant / 504-Web page / \n",
"3-Concert / \n",
"3-Concert / 7-Musician / 16-Performance art / 465-Microphone / \n",
"5-Dance / 16-Performance art / \n",
"6-Animation / 15-Cartoon / \n",
"3-Concert / \n",
"0-Games / 489-Board game / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 135-Drifting / \n",
"9-Music video / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 47-Personal computer / 145-Tablet computer / 621-Asus / \n",
"3-Concert / 16-Performance art / \n",
"1-Vehicle / 4-Car / 17-Racing / 67-Cycling / 69-Bicycle / 92-Radio-controlled model / 202-Radio-controlled car / \n",
"3-Concert / 5-Dance / \n",
"22-Nature / \n",
"12-Food / 25-Toy / 295-Chocolate / 582-Candy / 1408-Snack / \n",
"148-Painting / 998-Landscape / 3030-Landscape painting / \n",
"1-Vehicle / 4-Car / 1078-Installation art / 1119-Sensor / 1285-Parking / \n",
"3-Concert / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / \n",
"18-Outdoor recreation / 59-Winter sport / 112-Ice skating / 316-Figure skating / 632-Ice dancing / \n",
"18-Outdoor recreation / 96-Soldier / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / 592-Synthesia / \n",
"21-Stadium / 191-Cheerleading / 306-High school / 1662-Ōendan / \n",
"365-Fireworks / 1293-RollerCoaster Tycoon 3 / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 74-Truck / 945-MINI Cooper / \n",
"1-Vehicle / 4-Car / 84-Snow / 120-Winter / 198-Off-road vehicle / 406-Chevrolet / \n",
"0-Games / 2-Video game / 6-Animation / \n",
"3-Concert / 16-Performance art / 263-Light / \n",
"2-Video game / 5-Dance / 199-Wii / \n",
"0-Games / 1852-Final Fantasy XI / \n",
"2-Video game / 47-Personal computer / 131-Computer / 309-Computer hardware / 602-Central processing unit / \n",
"0-Games / 35-Minecraft / \n",
"117-Boxing / \n",
"0-Games / 240-Arcade game / 535-Slot machine / \n",
"46-Choir / 318-Church / 397-Organ / 870-Chapel / \n",
"0-Games / 530-Cue sports / \n",
"12-Food / 25-Toy / 200-Doll / 232-Cake / 1365-Fondant icing / \n",
"102-Building / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"3-Concert / 1684-Hillsong Church / \n",
"147-River / 164-Beach / 722-Waterfall / \n",
"6-Animation / 15-Cartoon / 20-Trailer / 126-The Walt Disney Company / \n",
"56-Hair / 106-Hairstyle / \n",
"325-Vampire / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / 265-Electronic keyboard / \n",
"0-Games / 2-Video game / 511-Mortal Kombat / 1291-Mortal Kombat / 4684-Kung Lao / \n",
"28-Fashion / 45-Cosmetics / 785-Jeans / \n",
"6-Animation / 2110-D.Gray-man / \n",
"110-Album / \n",
"18-Outdoor recreation / 59-Winter sport / 112-Ice skating / 316-Figure skating / \n",
"15-Cartoon / 477-Transformers / 3398-Transformers: Generation 1 / 3795-Starscream / \n",
"102-Building / 113-House / 310-The Sims / 328-Simulation video game / 564-The Sims 3 / \n",
"0-Games / 19-PC game / 51-Strategy video game / 165-League of Legends / \n",
"170-Home improvement / 276-Room / \n",
"113-House / \n",
"219-Accordion / \n",
"12-Food / 2080-Cup / \n",
"61-Art / 402-Jewellery / 829-Bead / 859-Beadwork / \n",
"9-Music video / \n",
"2-Video game / 33-Weapon / 55-Video game console / 2034-Dreamcast / \n",
"516-Headphones / \n",
"10-Animal / 424-Chicken / \n",
"6-Animation / 15-Cartoon / 183-Manga / 801-Inuyasha / \n",
"125-Television / 1348-Panasonic Corporation / 3984-Plasma display / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / \n",
"419-Earth / 448-Planet / 593-Moon / 616-The Bible / 1480-Astronaut / 1686-Orbit / \n",
"9-Music video / \n",
"20-Trailer / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 46-Choir / \n",
"12-Food / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 18-Outdoor recreation / 74-Truck / 167-Four-wheel drive / 198-Off-road vehicle / 277-Sport utility vehicle / 375-Pickup truck / 403-Toyota / 1840-Toyota Hilux / \n",
"5-Dance / 16-Performance art / 115-Ballet / \n",
"1-Vehicle / \n",
"121-Photography / 155-Camera / 566-Digital camera / 629-Camera lens / 721-Digital SLR / 3258-Leica Camera / \n",
"1-Vehicle / 4-Car / \n",
"0-Games / 8-Football / 21-Stadium / 73-Ball / \n",
"110-Album / \n",
"121-Photography / 155-Camera / 2248-Zoom lens / \n",
"93-Comedy / \n",
"2-Video game / 6-Animation / 15-Cartoon / 320-Super Smash Bros. / 398-The Legend of Zelda / 692-Link / 697-Super Smash Bros. for Nintendo 3DS and Wii U / \n",
"31-Disc jockey / \n",
"31-Disc jockey / \n",
"15-Cartoon / 116-Pokémon / 186-Pokémon / \n",
"6-Animation / 20-Trailer / 264-Star Wars / 714-Jedi / 717-Darth Vader / 2576-Clone trooper / \n",
"0-Games / 1721-Tetris / \n",
"12-Food / 26-Cooking / 32-Recipe / 122-Weight training / 542-Potato / 780-Fat / 1283-Pancake / 2575-Sweet potato / 2995-Blueberry / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 299-Volkswagen Passenger Cars / 1498-Key / \n",
"2-Video game / 229-Halo / 470-Halo 3 / 644-Halo: Reach / 866-Halo 4 / 2889-343 Industries / \n",
"3-Concert / \n",
"5-Dance / 13-Musical ensemble / 16-Performance art / \n",
"3-Concert / 5-Dance / 16-Performance art / 889-Swing / \n",
"1-Vehicle / 490-Rain / \n",
"7-Musician / 14-Guitar / 24-String instrument / 376-Flute / \n",
"6-Animation / 15-Cartoon / \n",
"12-Food / 720-Wine / \n",
"12-Food / \n",
"0-Games / 2-Video game / \n",
"30-Drummer / 39-Drums / 44-Drums / 123-Naruto / 128-Cymbal / 146-Snare drum / \n",
"237-Final Fantasy / 2204-Final Fantasy VI / \n",
"66-Bollywood / \n",
"10-Animal / 22-Nature / 48-Pet / 71-Dog / \n",
"22-Nature / 213-Forest / 423-Coast / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"9-Music video / \n",
"5-Dance / \n",
"42-Fishing / 1306-Carp fishing / \n",
"18-Outdoor recreation / 4695-Strikeout / \n",
"47-Personal computer / 1045-Computer case / \n",
"0-Games / 88-Machine / 240-Arcade game / 535-Slot machine / \n",
"20-Trailer / 3395-Asterix / \n",
"35-Minecraft / \n",
"10-Animal / 80-Horse / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 24-String instrument / 38-Orchestra / 150-Violin / 401-Cello / 450-Viola / \n",
"0-Games / 51-Strategy video game / 373-Clash of Clans / 1001-Gemstone / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 141-Microsoft Windows / \n",
"6-Animation / 15-Cartoon / \n",
"98-Festival / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 1286-Subaru / 3679-Super GT / \n",
"60-Basketball / \n",
"5-Dance / 16-Performance art / 678-Latin dance / \n",
"6-Animation / 15-Cartoon / \n",
"293-Slide show / \n",
"0-Games / 1604-Satan / \n",
"3-Concert / 16-Performance art / \n",
"6-Animation / 708-Mermaid / \n",
"1-Vehicle / 4-Car / 74-Truck / 158-Tire / \n",
"3-Concert / \n",
"6-Animation / \n",
"2952-Ventriloquism / \n",
"3-Concert / 13-Musical ensemble / 597-Soup / \n",
"8-Football / \n",
"2-Video game / 47-Personal computer / 908-Video card / \n",
"1-Vehicle / 62-Train / 64-Transport / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / 1781-Ollie / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 42-Fishing / 91-Fish / 1306-Carp fishing / \n",
"3-Concert / \n",
"522-Helmet / 748-Iron Man / \n",
"8-Football / \n",
"162-Medicine / \n",
"0-Games / 2-Video game / 51-Strategy video game / 165-League of Legends / \n",
"260-Prayer / \n",
"466-Gift / \n",
"1-Vehicle / 4-Car / 114-Engine / 154-Wheel / 403-Toyota / 2256-Bearing / 3399-Timing belt / 3685-Pulley / \n",
"7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"10-Animal / 537-Zoo / 970-Monkey / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 96-Soldier / 194-Call of Duty: Modern Warfare 2 / 276-Room / 1892-Dormitory / 2879-Medal of Honor: Warfighter / \n",
"2-Video game / 237-Final Fantasy / 1131-Final Fantasy XIII / \n",
"10-Animal / 48-Pet / 71-Dog / 1278-Labrador Retriever / \n",
"66-Bollywood / 113-House / 339-Architecture / 888-Living room / \n",
"2-Video game / 55-Video game console / \n",
"6-Animation / 290-Pony / 343-My Little Pony / \n",
"5-Dance / 342-Windows Media Video / \n",
"5-Dance / 98-Festival / 250-Parade / 253-Carnival / \n",
"429-Website / 819-World Wide Web / \n",
"1584-Kettlebell / \n",
"3-Concert / \n",
"8-Football / \n",
"217-Hunting / 882-Pen / 1873-Crossbow / \n",
"28-Fashion / \n",
"66-Bollywood / \n",
"3-Concert / 16-Performance art / 30-Drummer / \n",
"5-Dance / 8-Football / 21-Stadium / 4106-Moonwalk / \n",
"3-Concert / 13-Musical ensemble / \n",
"5-Dance / 16-Performance art / 95-Talent show / \n",
"2996-Caterpillar / \n",
"10-Animal / 48-Pet / 1147-Hamster / 1234-Mouse / 1716-Rat / \n",
"31-Disc jockey / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 42-Fishing / 275-Diving / 287-Human swimming / 288-Underwater / 1135-Dolphin / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"9-Music video / \n",
"415-Sewing / \n",
"10-Animal / 80-Horse / 247-Stallion / 438-Mare / \n",
"1222-Fountain / \n",
"0-Games / 2-Video game / 55-Video game console / 1118-Kirby / 1133-Kirby / \n",
"1-Vehicle / 4-Car / \n",
"5-Dance / 9-Music video / 31-Disc jockey / \n",
"1-Vehicle / 4-Car / 395-Camping / \n",
"1-Vehicle / 4-Car / \n",
"783-Zee TV / \n",
"0-Games / 2-Video game / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / 1860-Sonic Adventure / 1890-Dr. Eggman / 2034-Dreamcast / \n",
"0-Games / 35-Minecraft / \n",
"0-Games / 54-Highlight film / 60-Basketball / 168-Basketball moves / \n",
"1-Vehicle / 87-Boat / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 114-Engine / 332-Mercedes-Benz / 854-Diesel engine / \n",
"10-Animal / 571-Bear / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / \n",
"0-Games / 8-Football / 65-Kick / \n",
"25-Toy / 113-House / 200-Doll / 323-Toddler / 652-Bathroom / 1113-Toilet / 2153-Sink / 2861-Baby Alive / \n",
"0-Games / 2-Video game / 1251-The Witcher / 2497-The Witcher 2: Assassins of Kings / \n",
"3-Concert / 16-Performance art / \n",
"7-Musician / 13-Musical ensemble / 21-Stadium / 79-American football / 203-Marching band / 209-University / 250-Parade / \n",
"1-Vehicle / 50-Aircraft / 139-Airport / 151-Landing / 175-Airliner / 188-Airline / 254-Runway / 472-Boeing 737 / 1736-Microsoft Flight Simulator / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / \n",
"36-Piano / \n",
"8-Football / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 118-Grand Theft Auto V / 176-PlayStation 4 / 297-Grand Theft Auto IV / 317-PlayStation / \n",
"88-Machine / 521-Metal / 943-Stove / 2331-Melting / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 212-Highway / 947-Muffler / \n",
"0-Games / 2-Video game / 19-PC game / 55-Video game console / 292-Handheld game console / 320-Super Smash Bros. / \n",
"1-Vehicle / 4-Car / 158-Tire / 299-Volkswagen Passenger Cars / 668-Volkswagen Golf / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / \n",
"20-Trailer / \n",
"1090-Canoe / 1305-Canoeing / 2273-Polo / \n",
"2848-12-inch single / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 44-Drums / \n",
"0-Games / 2-Video game / 111-PlayStation 3 / 127-Xbox 360 / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / 1512-Sonic Generations / \n",
"45-Cosmetics / 56-Hair / 173-Eye shadow / 195-Mascara / 291-Rouge / \n",
"0-Games / 35-Minecraft / \n",
"75-Wrestling / \n",
"28-Fashion / \n",
"1-Vehicle / 27-Motorcycle / 493-Rim / \n",
"0-Games / 326-Metin2 / \n",
"3-Concert / 46-Choir / \n",
"5-Dance / \n",
"93-Comedy / \n",
"9-Music video / \n",
"22-Nature / 42-Fishing / 233-Ocean / 1075-Whale / 1135-Dolphin / 3797-Humpback whale / \n",
"1-Vehicle / 11-Motorsport / 124-Tractor / 174-Farm / 249-Heavy equipment / 252-Tractor pulling / 817-Bulldozer / \n",
"7-Musician / 9-Music video / 14-Guitar / \n",
"840-Museum / \n",
"0-Games / 2-Video game / 229-Halo / 470-Halo 3 / 644-Halo: Reach / \n",
"4076-Ultima / \n",
"28-Fashion / 56-Hair / 106-Hairstyle / 1359-Hair iron / \n",
"61-Art / 148-Painting / 269-Comic book / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / 387-Spider-Man / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"6-Animation / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 226-Dessert / 662-Pizza / \n",
"5-Dance / \n",
"0-Games / 79-American football / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 428-Rail transport modelling / \n",
"6-Animation / 15-Cartoon / \n",
"75-Wrestling / 85-Combat / \n",
"3-Concert / 7-Musician / 4089-Maracas / \n",
"6-Animation / 25-Toy / 1850-Playmobil / \n",
"9-Music video / \n",
"2-Video game / 20-Trailer / \n",
"5-Dance / 66-Bollywood / \n",
"23-Mobile phone / 29-Smartphone / 101-Telephone / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"85-Combat / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 135-Drifting / \n",
"130-Gymnastics / \n",
"20-Trailer / \n",
"12-Food / 2329-Felt / \n",
"20-Trailer / \n",
"10-Animal / 48-Pet / 71-Dog / \n",
"0-Games / 2-Video game / 1566-Doom / 2352-Doom II: Hell on Earth / \n",
"0-Games / 2-Video game / 20-Trailer / 43-Call of Duty / 422-Call of Duty: Ghosts / \n",
"10-Animal / 86-Plant / 2284-Bamboo / \n",
"0-Games / 35-Minecraft / 102-Building / 113-House / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 30-Drummer / \n",
"0-Games / 2-Video game / 51-Strategy video game / \n",
"0-Games / 2-Video game / 910-Dragon Quest / \n",
"0-Games / 6-Animation / 460-Yu-Gi-Oh! Trading Card Game / \n",
"356-Star / 502-Sky / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"122-Weight training / 162-Medicine / 214-Muscle / \n",
"18-Outdoor recreation / 59-Winter sport / 112-Ice skating / 262-Ice / 316-Figure skating / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"10-Animal / 846-Squirrel / \n",
"219-Accordion / 655-Diatonic button accordion / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / \n",
"240-Arcade game / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / \n",
"3-Concert / 5-Dance / 16-Performance art / 99-Christmas / \n",
"56-Hair / 106-Hairstyle / \n",
"6-Animation / 242-Paper / 295-Chocolate / 466-Gift / \n",
"0-Games / 8-Football / 21-Stadium / 73-Ball / \n",
"45-Cosmetics / 49-School / \n",
"3-Concert / 13-Musical ensemble / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / 1971-Epiphone / \n",
"14-Guitar / 24-String instrument / \n",
"1-Vehicle / 4-Car / \n",
"764-The King of Fighters / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / 336-Scooter / \n",
"0-Games / 75-Wrestling / \n",
"102-Building / \n",
"28-Fashion / 153-Dress / 495-Runway / \n",
"61-Art / 97-Drawing / 538-Illustration / \n",
"110-Album / \n",
"164-Beach / \n",
"1751-Image scanner / \n",
"425-City / \n",
"227-Water / 827-Pump / 2635-Water well / \n",
"0-Games / 23-Mobile phone / 29-Smartphone / 178-Samsung Galaxy / \n",
"5-Dance / 16-Performance art / 769-Red Bull / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / \n",
"0-Games / 47-Personal computer / 141-Microsoft Windows / 228-Laptop / \n",
"1-Vehicle / 4-Car / 11-Motorsport / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 24-String instrument / 38-Orchestra / 150-Violin / 401-Cello / 450-Viola / 545-Quartet (ensemble) / 726-Clarinet / 1002-String quartet / \n",
"0-Games / 2-Video game / 19-PC game / 47-Personal computer / \n",
"1-Vehicle / 18-Outdoor recreation / 87-Boat / 366-Dragon / 949-Rowing / 2678-Dragon boat / \n",
"0-Games / 2-Video game / 6-Animation / 10-Animal / 290-Pony / 343-My Little Pony / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / 1669-Vodka / \n",
"14-Guitar / 2120-Tears / \n",
"1-Vehicle / 4-Car / 17-Racing / \n",
"3-Concert / 5-Dance / 16-Performance art / 95-Talent show / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"0-Games / 59-Winter sport / 81-Athlete / 112-Ice skating / 142-Hockey / 312-Ice rink / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 19-PC game / 123-Naruto / 285-Sasuke Uchiha / 591-Naruto: Ultimate Ninja / \n",
"9-Music video / \n",
"36-Piano / \n",
"3-Concert / 16-Performance art / 46-Choir / \n",
"4400-Rain gutter / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 452-Watch / 1260-Smartwatch / 3727-Android Wear / \n",
"23-Mobile phone / 29-Smartphone / 345-Loudspeaker / \n",
"0-Games / 2-Video game / 237-Final Fantasy / 730-Final Fantasy VII / \n",
"9-Music video / \n",
"0-Games / 211-RuneScape / \n",
"0-Games / 2-Video game / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / 2869-Bomberman / \n",
"359-Harry Potter (Literary Series) / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / \n",
"5-Dance / \n",
"31-Disc jockey / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / \n",
"3-Concert / \n",
"6-Animation / 15-Cartoon / 440-Lion / \n",
"0-Games / 1182-Softball / \n",
"133-Nightclub / 613-Bar / \n",
"455-Unidentified flying object / \n",
"3-Concert / 16-Performance art / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 40-Road / 105-Motorcycling / \n",
"0-Games / 2-Video game / 2944-La Tale / \n",
"1222-Fountain / \n",
"107-Musical keyboard / \n",
"1035-Bank / 1528-Euro / \n",
"1-Vehicle / 4-Car / \n",
"1-Vehicle / \n",
"2531-Dojo / \n",
"0-Games / 2-Video game / 8-Football / 90-Sports game / 180-FIFA 15 / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / \n",
"18-Outdoor recreation / 22-Nature / 147-River / 722-Waterfall / \n",
"1-Vehicle / 4-Car / 114-Engine / 1575-Clutch / \n",
"2-Video game / \n",
"22-Nature / 36-Piano / \n",
"93-Comedy / 341-Sketch comedy / 364-Sitcom / \n",
"5-Dance / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 105-Motorcycling / 197-Trail / \n",
"0-Games / 90-Sports game / 180-FIFA 15 / 848-FIFA 12 / \n",
"9-Music video / \n",
"122-Weight training / 214-Muscle / \n",
"0-Games / 2-Video game / 432-Kingdom Hearts / 4250-Kingdom Hearts 3D: Dream Drop Distance / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / \n",
"66-Bollywood / 905-Flash Video / \n",
"3-Concert / 7-Musician / \n",
"2-Video game / 89-Comics / 2172-Deadpool / \n",
"5-Dance / 98-Festival / 203-Marching band / \n",
"9-Music video / \n",
"365-Fireworks / 942-Silkroad Online / \n",
"10-Animal / 18-Outdoor recreation / 217-Hunting / 3022-Hare / \n",
"0-Games / 60-Basketball / \n",
"2042-Nut / \n",
"466-Gift / 627-Clock / \n",
"102-Building / 113-House / \n",
"3-Concert / \n",
"9-Music video / \n",
"3063-Friendship bracelet / \n",
"1572-Husband / \n",
"9-Music video / 22-Nature / 283-Rock / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / 44-Drums / \n",
"1-Vehicle / 62-Train / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / 1259-Fender Telecaster / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 47-Personal computer / 127-Xbox 360 / 181-Xbox / 383-Battlefield 4 / 677-Game controller / 704-Call of Duty: World at War / 1248-Call of Duty: Zombies / 1872-Xbox 360 Controller / \n",
"12-Food / 367-Drink / 613-Bar / 904-Cocktail / 1065-Bartender / \n",
"9-Music video / \n",
"8-Football / 924-Trophy / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 197-Trail / 213-Forest / \n",
"3-Concert / 3044-Sesame / \n",
"93-Comedy / 1369-Necktie / \n",
"7-Musician / 30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / 4131-Mapex Drums / \n",
"5-Dance / 98-Festival / \n",
"5-Dance / 16-Performance art / 95-Talent show / 2198-Irish dance / \n",
"1-Vehicle / 4-Car / \n",
"12-Food / 32-Recipe / 236-Baking / 1408-Snack / 2046-Popcorn / \n",
"6-Animation / 15-Cartoon / 411-MapleStory / \n",
"319-Robot / \n",
"0-Games / 8-Football / \n",
"309-Computer hardware / \n",
"0-Games / 2-Video game / 116-Pokémon / 186-Pokémon / 1387-Pokémon Battle Revolution / \n",
"9-Music video / \n",
"0-Games / 2-Video game / \n",
"367-Drink / 612-Beer / \n",
"238-Nail / 300-Nail art / 303-Paint / 304-Nail polish / 307-Hand / 324-Finger / 338-Manicure / 873-Acrylic paint / 936-Watercolor paint / \n",
"937-Black belt / \n",
"7-Musician / 9-Music video / 13-Musical ensemble / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 302-Call of Duty 4: Modern Warfare / \n",
"2-Video game / 23-Mobile phone / 47-Personal computer / 145-Tablet computer / \n",
"20-Trailer / \n",
"1-Vehicle / 4-Car / 277-Sport utility vehicle / 298-Television advertisement / 4493-Nissan X-Trail / \n",
"10-Animal / 280-Wildlife / 537-Zoo / 999-Elephant / \n",
"59-Winter sport / 84-Snow / 120-Winter / 177-Skiing / 281-Snowboarding / \n",
"0-Games / 2-Video game / 51-Strategy video game / 165-League of Legends / \n",
"1-Vehicle / 4-Car / 40-Road / 94-Dashcam / \n",
"0-Games / 544-Table tennis / \n",
"8-Football / \n",
"804-Shopping mall / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 635-Achievement / 948-Far Cry / \n",
"0-Games / 1680-Seafight / \n",
"1-Vehicle / 10-Animal / 87-Boat / 225-Lake / \n",
"5-Dance / 31-Disc jockey / 78-Wedding / 121-Photography / 2269-Wedding photography / \n",
"51-Strategy video game / 321-Roller coaster / 334-Amusement ride / 1293-RollerCoaster Tycoon 3 / 3620-RollerCoaster Tycoon / \n",
"1-Vehicle / 27-Motorcycle / 207-Exhaust system / \n",
"1-Vehicle / 27-Motorcycle / 197-Trail / 213-Forest / \n",
"460-Yu-Gi-Oh! Trading Card Game / \n",
"12-Food / 3250-Chubby Bunny / \n",
"0-Games / 2-Video game / 34-Action-adventure game / \n",
"14-Guitar / 100-Electric guitar / \n",
"10-Animal / 12-Food / 26-Cooking / 52-Dish / 58-Cuisine / 215-Eating / 424-Chicken / 1438-Street food / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 247-Stallion / 289-Dressage / 404-Livestock / 4322-Dutch Warmblood / \n",
"3-Concert / 16-Performance art / 253-Carnival / \n",
"3-Concert / 14-Guitar / 502-Sky / \n",
"31-Disc jockey / \n",
"0-Games / 1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 169-Rallying / 711-Gran Turismo / 2476-Maze / \n",
"20-Trailer / \n",
"117-Boxing / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 266-Helicopter / 392-Radio-controlled helicopter / \n",
"20-Trailer / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 192-Battlefield / 374-Battlefield 3 / 383-Battlefield 4 / \n",
"273-News program / \n",
"689-Harmonica / \n",
"0-Games / 211-RuneScape / \n",
"170-Home improvement / 248-Hotel / \n",
"743-Edward Cullen / \n",
"7-Musician / 36-Piano / 150-Violin / 401-Cello / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / 2435-Tokyo Ghoul / \n",
"2118-Tongue / \n",
"3-Concert / \n",
"1-Vehicle / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 233-Ocean / 278-GoPro / 393-Unmanned aerial vehicle / \n",
"3-Concert / 30-Drummer / 39-Drums / \n",
"9-Music video / 330-Money / \n",
"0-Games / 2-Video game / 511-Mortal Kombat / 635-Achievement / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 189-iPad / 272-iPod / 331-iPod touch / \n",
"28-Fashion / 45-Cosmetics / 246-Furniture / 276-Room / 580-Bedroom / \n",
"282-Skin / 3861-Toner / \n",
"0-Games / 51-Strategy video game / 165-League of Legends / \n",
"0-Games / 1-Vehicle / 2-Video game / 34-Action-adventure game / 50-Aircraft / 201-Grand Theft Auto: San Andreas / \n",
"0-Games / 1-Vehicle / 2-Video game / 11-Motorsport / 1817-Initial D Arcade Stage / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / \n",
"61-Art / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 189-iPad / 272-iPod / 1899-E-book / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"10-Animal / 1234-Mouse / \n",
"463-Cricket / 1476-Bowling (cricket) / \n",
"5-Dance / 889-Swing / \n",
"238-Nail / 1325-Gel / \n",
"10-Animal / 22-Nature / 166-Cat / \n",
"12-Food / 1208-Coconut / \n",
"125-Television / \n",
"5-Dance / 16-Performance art / 4089-Maracas / \n",
"36-Piano / 164-Beach / 978-Love song / \n",
"22-Nature / 61-Art / 147-River / 148-Painting / 227-Water / \n",
"0-Games / 2-Video game / 19-PC game / 47-Personal computer / 131-Computer / 192-Battlefield / 383-Battlefield 4 / \n",
"31-Disc jockey / \n",
"819-World Wide Web / \n",
"3-Concert / 68-Lighting / \n",
"7-Musician / 36-Piano / 104-Pianist / \n",
"5-Dance / 49-School / \n",
"9-Music video / \n",
"0-Games / 159-World of Warcraft / 171-Warcraft / 1356-Paladin / 1882-Warrior / \n",
"25-Toy / 140-LEGO / 884-Lego minifigure / \n",
"67-Cycling / 69-Bicycle / 377-BMX bike / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 157-Jet aircraft / 388-Fighter aircraft / 1483-United States Air Force / 2954-McDonnell Douglas F-15 Eagle / \n",
"0-Games / 2-Video game / 19-PC game / 928-Borderlands 2 / \n",
"293-Slide show / \n",
"1-Vehicle / 4-Car / 344-Coupé / 1314-Garage / 2639-Automobile repair shop / \n",
"47-Personal computer / 228-Laptop / \n",
"6-Animation / 547-Bleach / 3562-Hollow / \n",
"0-Games / 49-School / 54-Highlight film / 60-Basketball / \n",
"9-Music video / \n",
"98-Festival / 110-Album / \n",
"1-Vehicle / 4-Car / 11-Motorsport / \n",
"0-Games / \n",
"624-Number / 1546-Line / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 89-Comics / 185-Dragon Ball / 244-Goku / 418-Gohan / 869-Dragon Ball Z: Budokai Tenkaichi / 1244-Trunks / 4335-Dragon Ball Z: Burst Limit / \n",
"0-Games / 2-Video game / 6-Animation / 20-Trailer / 55-Video game console / 111-PlayStation 3 / 176-PlayStation 4 / 396-One Piece / \n",
"6-Animation / 1199-Kinect / \n",
"1-Vehicle / 4-Car / 1046-Remote control / 1245-Alarm device / \n",
"45-Cosmetics / 235-Lipstick / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"150-Violin / \n",
"3-Concert / 7-Musician / \n",
"28-Fashion / 206-Shoe / 757-Boot / \n",
"22-Nature / 84-Snow / 800-Mountain pass / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / 207-Exhaust system / 806-Chopper / 4170-Honda Shadow / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 542-Potato / 2175-Lamb and mutton / 3384-Cutlet / \n",
"36-Piano / 1555-Sword Art Online / \n",
"3-Concert / 5-Dance / 671-Tomato / \n",
"1-Vehicle / 62-Train / 1279-Plarail / \n",
"121-Photography / \n",
"0-Games / 49-School / 60-Basketball / \n",
"122-Weight training / \n",
"20-Trailer / \n",
"1113-Toilet / \n",
"22-Nature / 164-Beach / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 76-Ball / 81-Athlete / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / \n",
"0-Games / 2-Video game / 19-PC game / 1187-Ragnarok Online / \n",
"12-Food / \n",
"1-Vehicle / 4-Car / 74-Truck / 2007-Intermodal container / \n",
"1-Vehicle / 4-Car / \n",
"5-Dance / \n",
"20-Trailer / 222-Surfing / 357-DVD / 815-Surfboard / \n",
"3-Concert / 16-Performance art / 3389-West Side Story / \n",
"8-Football / \n",
"14-Guitar / 100-Electric guitar / \n",
"3-Concert / 16-Performance art / \n",
"10-Animal / 987-Sheep / \n",
"59-Winter sport / 84-Snow / 177-Skiing / 305-Ski / \n",
"1-Vehicle / 18-Outdoor recreation / 22-Nature / 40-Road / 283-Rock / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 36-Piano / 38-Orchestra / 230-Brass instrument / 427-Trumpet / 634-Trombone / 1574-French horn / \n",
"10-Animal / 17-Racing / 384-Horse racing / \n",
"1-Vehicle / 4-Car / 1742-Lowrider / \n",
"0-Games / 1-Vehicle / 2-Video game / 11-Motorsport / 17-Racing / 27-Motorcycle / 172-Motocross / \n",
"0-Games / 1-Vehicle / 2-Video game / 34-Action-adventure game / 1275-Watch Dogs / \n",
"1786-Dam / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 179-Kitchen / 184-Vegetable / 301-Meal / 557-Salad / 742-Countertop / 1748-Carrot / \n",
"467-Chipmunk / \n",
"0-Games / 51-Strategy video game / 356-Star / 373-Clash of Clans / \n",
"59-Winter sport / 84-Snow / 120-Winter / 281-Snowboarding / 501-Snowboard / \n",
"10-Animal / 174-Farm / 723-Pig / 1123-Domestic pig / \n",
"2162-Marker pen / \n",
"12-Food / 86-Plant / 136-Gardening / 156-Agriculture / 174-Farm / 184-Vegetable / 279-Garden / 671-Tomato / 1384-Greenhouse / \n",
"260-Prayer / \n",
"13-Musical ensemble / 46-Choir / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"1363-Candle / \n",
"88-Machine / 208-Wood / 261-Tool / 400-Woodturning / 640-Bottle / 972-Lathe / \n",
"6-Animation / 701-Beyblade / 1004-Samurai / 3868-Beyblade: Shogun Steel / \n",
"36-Piano / 53-Keyboard / 397-Organ / 646-Pipe organ / \n",
"25-Toy / 140-LEGO / 313-Book / 434-Ninja / 884-Lego minifigure / 1478-Lego Ninjago / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 172-Motocross / \n",
"36-Piano / 53-Keyboard / 104-Pianist / \n",
"298-Television advertisement / 337-Advertising / \n",
"3-Concert / 7-Musician / 30-Drummer / 44-Drums / \n",
"0-Games / 6-Animation / 460-Yu-Gi-Oh! Trading Card Game / \n",
"0-Games / 47-Personal computer / 55-Video game console / 111-PlayStation 3 / \n",
"1-Vehicle / 11-Motorsport / 22-Nature / 198-Off-road vehicle / 213-Forest / 327-Mud / 565-Mud bogging / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 77-Arena / \n",
"125-Television / \n",
"102-Building / 113-House / 170-Home improvement / 208-Wood / 707-Roof / 879-Ceiling / \n",
"126-The Walt Disney Company / 144-Amusement park / 576-Walt Disney World / \n",
"0-Games / 35-Minecraft / \n",
"59-Winter sport / 83-Skateboarding / 108-Skateboard / 112-Ice skating / \n",
"25-Toy / 200-Doll / 786-Juice / 2861-Baby Alive / \n",
"25-Toy / 476-Textile / 2914-Wreath / \n",
"414-Computer monitor / 621-Asus / \n",
"0-Games / 2-Video game / 43-Call of Duty / 134-Call of Duty: Black Ops / 194-Call of Duty: Modern Warfare 2 / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 247-Stallion / 289-Dressage / 404-Livestock / 438-Mare / 1434-Saddle / \n",
"0-Games / 85-Combat / 117-Boxing / \n",
"452-Watch / \n",
"1-Vehicle / 4-Car / 154-Wheel / \n",
"12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / 210-Cookware and bakeware / 236-Baking / 442-Bread / 485-Dough / 578-Cookie / 1283-Pancake / 2995-Blueberry / \n",
"1-Vehicle / 4-Car / \n",
"3-Concert / 5-Dance / 7-Musician / 30-Drummer / 133-Nightclub / \n",
"9-Music video / 78-Wedding / 190-Bride / 318-Church / \n",
"3071-Towel / \n",
"99-Christmas / 242-Paper / 370-Origami / \n",
"22-Nature / 499-Cloud / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 3586-F1 2011 / 3731-F1 2010 / \n",
"725-Human / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 178-Samsung Galaxy / 487-Samsung Electronics / 1015-Samsung Galaxy Note series / 1776-Samsung Galaxy Note II / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 31-Disc jockey / 135-Drifting / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 96-Soldier / 192-Battlefield / 3450-Battlefield 1942 / \n",
"1-Vehicle / 205-Newscaster / 273-News program / \n",
"439-Barbie / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 301-Meal / \n",
"6-Animation / 123-Naruto / 434-Ninja / 2169-Ninja World / \n",
"1-Vehicle / 2-Video game / 4-Car / 70-Driving / 1758-Test Drive / 1766-Test Drive Unlimited / 2113-Test Drive Unlimited 2 / \n",
"5-Dance / 16-Performance art / 93-Comedy / \n",
"2-Video game / 47-Personal computer / 294-Batman / 464-First-person Shooter / 1261-Batman: Arkham City / \n",
"6-Animation / 25-Toy / 435-Action figure / \n",
"6-Animation / \n",
"0-Games / 2-Video game / 55-Video game console / 2101-Ace Combat / \n",
"23-Mobile phone / 29-Smartphone / 101-Telephone / 465-Microphone / 802-Google Nexus / \n",
"7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / \n",
"1459-Magazine / \n",
"122-Weight training / 214-Muscle / 1055-Back / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / \n",
"5-Dance / 115-Ballet / 3092-Cinderella / \n",
"0-Games / 8-Football / \n",
"9-Music video / 256-Tennis / \n",
"1-Vehicle / 4-Car / 164-Beach / \n",
"20-Trailer / \n",
"6-Animation / 15-Cartoon / 126-The Walt Disney Company / 2076-Goofy / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 2555-Harpsichord / \n",
"6-Animation / \n",
"3-Concert / 7-Musician / \n",
"28-Fashion / 49-School / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"88-Machine / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 92-Radio-controlled model / 129-Aviation / 149-Model aircraft / 160-Radio-controlled aircraft / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 179-Kitchen / 226-Dessert / 232-Cake / 295-Chocolate / 684-Ice cream / \n",
"0-Games / 10-Animal / 15-Cartoon / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / 1031-Fisheye lens / \n",
"0-Games / 8-Football / \n",
"0-Games / 2-Video game / 96-Soldier / 192-Battlefield / 374-Battlefield 3 / 383-Battlefield 4 / \n",
"0-Games / 2-Video game / \n",
"1-Vehicle / 11-Motorsport / 124-Tractor / 156-Agriculture / 252-Tractor pulling / 1129-Hay / \n",
"5-Dance / 2449-Chuck E. Cheese's / \n",
"49-School / \n",
"139-Airport / 1013-Walking / \n",
"5-Dance / 16-Performance art / \n",
"293-Slide show / \n",
"18-Outdoor recreation / 347-Climbing / 828-Rock climbing / \n",
"0-Games / 194-Call of Duty: Modern Warfare 2 / \n",
"3-Concert / \n",
"0-Games / 2-Video game / \n",
"5-Dance / \n",
"20-Trailer / \n",
"260-Prayer / 1604-Satan / \n",
"0-Games / 2-Video game / 586-Nintendo Entertainment System / \n",
"2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / \n",
"3-Concert / \n",
"3031-Aragorn / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 19-PC game / 192-Battlefield / 374-Battlefield 3 / \n",
"3-Concert / 7-Musician / \n",
"3-Concert / 5-Dance / 49-School / 95-Talent show / 191-Cheerleading / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 67-Cycling / 69-Bicycle / 377-BMX bike / \n",
"12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / 109-Cooking show / 1022-Chili pepper / 2609-Canning / 4266-Jalapeño / \n",
"1-Vehicle / 10-Animal / 42-Fishing / 87-Boat / 91-Fish / \n",
"3-Concert / 7-Musician / 36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"20-Trailer / \n",
"10-Animal / 84-Snow / 217-Hunting / 706-Deer / \n",
"9-Music video / 20-Trailer / 110-Album / \n",
"0-Games / 1029-Aion: The Tower of Eternity / \n",
"0-Games / 2-Video game / 464-First-person Shooter / 1749-Titanfall / \n",
"693-Sword / \n",
"452-Watch / 619-Manufacturing / 627-Clock / \n",
"1-Vehicle / 4-Car / 3197-Citroën C4 / \n",
"86-Plant / 279-Garden / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"0-Games / 310-The Sims / 328-Simulation video game / 543-The Sims 2 / \n",
"8-Football / \n",
"6-Animation / 25-Toy / 435-Action figure / \n",
"2816-Billboard / \n",
"15-Cartoon / 165-League of Legends / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 79-American football / \n",
"5-Dance / \n",
"99-Christmas / \n",
"46-Choir / \n",
"5-Dance / 16-Performance art / 39-Drums / \n",
"37-Gadget / 47-Personal computer / 145-Tablet computer / 189-iPad / 1150-iPad 2 / \n",
"162-Medicine / \n",
"66-Bollywood / \n",
"15-Cartoon / 903-Mass Effect / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 199-Wii / 3920-Maximum Destruction / \n",
"6-Animation / 15-Cartoon / 412-Monster / 503-Monster High / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 77-Arena / 81-Athlete / \n",
"10-Animal / 80-Horse / 247-Stallion / 289-Dressage / 290-Pony / \n",
"1-Vehicle / 4-Car / 64-Transport / 74-Truck / 249-Heavy equipment / 1611-Garbage truck / 3856-Landfill / \n",
"6-Animation / 577-Penguin / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 39-Drums / 230-Brass instrument / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 178-Samsung Galaxy / 1629-Samsung Galaxy Note / 1727-Samsung Galaxy Note 3 / \n",
"3-Concert / \n",
"2-Video game / 55-Video game console / 127-Xbox 360 / \n",
"5-Dance / 16-Performance art / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"12-Food / 26-Cooking / 52-Dish / 58-Cuisine / 2313-Bánh / \n",
"10-Animal / 22-Nature / 91-Fish / 258-Aquarium / \n",
"298-Television advertisement / 356-Star / \n",
"56-Hair / 106-Hairstyle / 607-Lace / 1292-Ponytail / \n",
"20-Trailer / 434-Ninja / \n",
"1-Vehicle / 62-Train / 2904-Express train / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"3-Concert / 38-Orchestra / 68-Lighting / \n",
"2-Video game / 43-Call of Duty / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / \n",
"3-Concert / 68-Lighting / \n",
"8-Football / 73-Ball / 76-Ball / \n",
"3-Concert / 13-Musical ensemble / \n",
"1-Vehicle / 4-Car / 270-Sedan / \n",
"36-Piano / 53-Keyboard / \n",
"9-Music video / \n",
"10-Animal / 42-Fishing / 91-Fish / 258-Aquarium / \n",
"6-Animation / 61-Art / 97-Drawing / 148-Painting / 183-Manga / 528-Pencil / 628-Portrait / \n",
"0-Games / 21-Stadium / 54-Highlight film / 81-Athlete / 562-Pitcher / 946-Baseball park / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / 265-Electronic keyboard / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 371-Firefighter / 663-Street / 796-Fire engine / 1633-Fire station / \n",
"5-Dance / 95-Talent show / \n",
"125-Television / \n",
"5-Dance / \n",
"9-Music video / \n",
"167-Four-wheel drive / 626-Puzzle / 631-Cube / 740-Rubik's Cube / \n",
"15-Cartoon / 31-Disc jockey / \n",
"5-Dance / 16-Performance art / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 76-Ball / 81-Athlete / 767-Goalkeeper / 1247-Penalty kick / \n",
"3-Concert / 30-Drummer / 68-Lighting / \n",
"10-Animal / 48-Pet / 132-Bird / 424-Chicken / 1136-Poultry / \n",
"162-Medicine / 1599-Bone / \n",
"22-Nature / 147-River / 283-Rock / 751-Desert / \n",
"0-Games / 8-Football / \n",
"1-Vehicle / 249-Heavy equipment / 822-Crane / 2194-Wind power / \n",
"1-Vehicle / 4-Car / 2826-Suzuki Swift / \n",
"6-Animation / 61-Art / 97-Drawing / 148-Painting / 1620-Pearl / \n",
"260-Prayer / \n",
"116-Pokémon / 330-Money / \n",
"5-Dance / 9-Music video / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / 813-Running back / \n",
"122-Weight training / 214-Muscle / 532-Barbell / 1055-Back / \n",
"0-Games / 6-Animation / 995-Attack on Titan / \n",
"1-Vehicle / 8-Football / 21-Stadium / 77-Arena / \n",
"278-GoPro / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / \n",
"150-Violin / 639-Fiddle / \n",
"12-Food / 226-Dessert / 232-Cake / 560-Sugar / 578-Cookie / 759-Icing / 795-Cupcake / 3641-Royal icing / \n",
"67-Cycling / 69-Bicycle / 221-Skatepark / 377-BMX bike / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / \n",
"1-Vehicle / 4-Car / 1321-Škoda Auto / 1540-Station Wagon / \n",
"0-Games / 2-Video game / 55-Video game console / \n",
"23-Mobile phone / 29-Smartphone / 121-Photography / 155-Camera / \n",
"971-Alphabet / \n",
"0-Games / 2-Video game / 6-Animation / 116-Pokémon / 186-Pokémon / 690-Pokémon X and Y / 1005-Pokémon Omega Ruby and Alpha Sapphire / \n",
"1-Vehicle / 25-Toy / 62-Train / 144-Amusement park / 507-Thomas the Tank Engine / \n",
"1-Vehicle / 27-Motorcycle / 207-Exhaust system / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 635-Achievement / 924-Trophy / \n",
"96-Soldier / \n",
"6-Animation / 15-Cartoon / 99-Christmas / 467-Chipmunk / \n",
"1-Vehicle / 11-Motorsport / 124-Tractor / 156-Agriculture / 174-Farm / 249-Heavy equipment / 252-Tractor pulling / 459-Plough / 660-Rural area / 3820-Harrow / \n",
"1-Vehicle / 82-Airplane / 114-Engine / 1263-Glider / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 172-Motocross / 252-Tractor pulling / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 169-Rallying / \n",
"0-Games / 2-Video game / 20-Trailer / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"12-Food / 26-Cooking / 58-Cuisine / \n",
"25-Toy / 1152-Plush / 2689-Furby / \n",
"1-Vehicle / 4-Car / 114-Engine / 449-Honda / 1614-Honda Accord / 3086-Honda CR-V / \n",
"5-Dance / 115-Ballet / \n",
"126-The Walt Disney Company / \n",
"0-Games / 2-Video game / 89-Comics / 241-Fighting game / 311-Street Fighter / \n",
"75-Wrestling / \n",
"97-Drawing / \n",
"9-Music video / \n",
"6-Animation / 15-Cartoon / \n",
"1-Vehicle / 87-Boat / 164-Beach / \n",
"23-Mobile phone / 29-Smartphone / 101-Telephone / 2929-Galaxy Nexus / \n",
"1-Vehicle / 5-Dance / \n",
"230-Brass instrument / 296-Saxophone / 1006-Tenor saxophone / \n",
"0-Games / 2-Video game / 75-Wrestling / 777-WWE 2K / 2629-WWE SmackDown vs. Raw 2010 / \n",
"6-Animation / 15-Cartoon / 1724-Homer Simpson / 3367-Bart Simpson / \n",
"303-Paint / 353-Wall / 488-Door / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 60-Basketball / 168-Basketball moves / \n",
"10-Animal / 12-Food / 523-Cattle / \n",
"0-Games / 14-Guitar / 445-Guitar Hero / 1144-Castlevania / \n",
"5-Dance / 16-Performance art / \n",
"3-Concert / 38-Orchestra / \n",
"126-The Walt Disney Company / 144-Amusement park / 576-Walt Disney World / \n",
"12-Food / 1363-Candle / \n",
"3-Concert / \n",
"9-Music video / \n",
"45-Cosmetics / 238-Nail / 300-Nail art / 304-Nail polish / 307-Hand / 324-Finger / 338-Manicure / 823-Artificial nails / \n",
"1-Vehicle / 12-Food / 124-Tractor / 156-Agriculture / 174-Farm / 984-Silage / \n",
"2-Video game / 43-Call of Duty / 394-Map / 431-Call of Duty: Advanced Warfare / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"0-Games / 19-PC game / 1187-Ragnarok Online / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 161-Train station / 224-Rapid transit / \n",
"3-Concert / 250-Parade / \n",
"402-Jewellery / 476-Textile / \n",
"222-Surfing / 672-Bodyboarding / \n",
"419-Earth / \n",
"67-Cycling / 69-Bicycle / 259-Fire / \n",
"309-Computer hardware / 908-Video card / \n",
"6-Animation / \n",
"8-Football / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 691-Cargo / 821-Rail freight transport / \n",
"402-Jewellery / 863-Body piercing / \n",
"0-Games / 2-Video game / 37-Gadget / 55-Video game console / 199-Wii / 292-Handheld game console / 443-Wii U / \n",
"0-Games / 1-Vehicle / 124-Tractor / 156-Agriculture / 174-Farm / 458-Gold / 715-Farming Simulator / \n",
"325-Vampire / 1950-The CW / \n",
"1-Vehicle / 62-Train / 64-Transport / 94-Dashcam / 152-Track / 161-Train station / 224-Rapid transit / \n",
"6-Animation / 15-Cartoon / \n",
"12-Food / 86-Plant / 348-Fruit / 2500-Pear / \n",
"9-Music video / \n",
"6-Animation / 15-Cartoon / 1884-Inazuma Eleven / 2450-Inazuma Eleven / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 51-Strategy video game / 271-Tank / 409-World of Tanks / \n",
"88-Machine / 1057-Embroidery / \n",
"205-Newscaster / 223-Weather / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 111-PlayStation 3 / 194-Call of Duty: Modern Warfare 2 / \n",
"0-Games / 1232-Perfect World / 2394-Perfect World / \n",
"25-Toy / 35-Minecraft / 140-LEGO / 3899-Lego Minecraft / \n",
"0-Games / 159-World of Warcraft / 171-Warcraft / 1213-World of Warcraft: Wrath of the Lich King / \n",
"1-Vehicle / 4-Car / 94-Dashcam / 449-Honda / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 226-Dessert / 232-Cake / 236-Baking / 534-Milk / 968-Pie / \n",
"60-Basketball / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 44-Drums / 689-Harmonica / \n",
"1-Vehicle / 4-Car / \n",
"0-Games / 51-Strategy video game / 382-Dota 2 / 549-Defense of the Ancients / \n",
"122-Weight training / 204-Gym / 214-Muscle / 529-Squat / 532-Barbell / \n",
"5-Dance / 986-Suit / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 38-Orchestra / 39-Drums / 44-Drums / 46-Choir / \n",
"5-Dance / \n",
"2-Video game / 47-Personal computer / 1996-Intel Core i7 / \n",
"1-Vehicle / 74-Truck / 249-Heavy equipment / 1429-Dump truck / \n",
"31-Disc jockey / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 70-Driving / 349-Chevrolet / 406-Chevrolet / 3793-Chevrolet Chevy II / Nova / \n",
"5-Dance / 66-Bollywood / \n",
"1-Vehicle / 286-Bus / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 163-Running / 201-Grand Theft Auto: San Andreas / 858-Carl Johnson / \n",
"49-School / \n",
"0-Games / 2-Video game / 33-Weapon / \n",
"7-Musician / 13-Musical ensemble / 49-School / 203-Marching band / 306-High school / \n",
"1-Vehicle / 10-Animal / 18-Outdoor recreation / 42-Fishing / 87-Boat / 91-Fish / 268-Recreational fishing / \n",
"31-Disc jockey / \n",
"1605-Second Life / \n",
"1-Vehicle / 648-Clown / \n",
"1-Vehicle / 11-Motorsport / 124-Tractor / 156-Agriculture / 249-Heavy equipment / 327-Mud / \n",
"3-Concert / 16-Performance art / 38-Orchestra / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 124-Tractor / 252-Tractor pulling / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"5-Dance / 115-Ballet / \n",
"0-Games / 35-Minecraft / \n",
"31-Disc jockey / \n",
"1-Vehicle / 18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / 336-Scooter / \n",
"164-Beach / 222-Surfing / \n",
"7-Musician / 13-Musical ensemble / 389-Restaurant / \n",
"0-Games / 2541-Allods Online / \n",
"516-Headphones / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 133-Nightclub / \n",
"9-Music video / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 79-American football / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 77-Arena / 81-Athlete / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / \n",
"250-Parade / \n",
"0-Games / 2-Video game / 1681-Mu Online / 4288-Carnage / \n",
"45-Cosmetics / 238-Nail / 300-Nail art / 304-Nail polish / 307-Hand / 324-Finger / 338-Manicure / 1128-Glitter / \n",
"85-Combat / 694-Tai chi / \n",
"39-Drums / 44-Drums / 1458-Electronic drum / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 635-Achievement / 924-Trophy / \n",
"12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / 1408-Snack / 2046-Popcorn / \n",
"3-Concert / 16-Performance art / \n",
"0-Games / 2-Video game / 6-Animation / 241-Fighting game / 514-Tekken / 797-Tekken / 1470-Tekken Tag Tournament 2 / 2812-Kazuya Mishima / \n",
"9-Music video / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 179-Kitchen / 354-Egg / \n",
"1-Vehicle / 198-Off-road vehicle / 741-Mower / \n",
"3-Concert / 16-Performance art / 99-Christmas / \n",
"1-Vehicle / 27-Motorcycle / \n",
"10-Animal / 48-Pet / 132-Bird / 598-Parrot / 1341-Parakeet / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 81-Athlete / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 82-Airplane / 118-Grand Theft Auto V / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / \n",
"1019-Christ / \n",
"0-Games / 19-PC game / 51-Strategy video game / 165-League of Legends / 1494-Jungle / \n",
"455-Unidentified flying object / \n",
"8-Football / 79-American football / \n",
"99-Christmas / \n",
"28-Fashion / 785-Jeans / 2315-Top / \n",
"1-Vehicle / 50-Aircraft / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / 472-Boeing 737 / 1529-Jet engine / 2064-Boeing 737 Next Generation / \n",
"0-Games / 33-Weapon / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 70-Driving / 234-Ford / 473-Ford Mustang / 647-Muscle car / 1361-Ford GT / \n",
"117-Boxing / \n",
"624-Number / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 234-Ford / 270-Sedan / 332-Mercedes-Benz / 473-Ford Mustang / 2753-SEAT León / \n",
"10-Animal / 22-Nature / 86-Plant / 723-Pig / 1215-Wild boar / \n",
"3-Concert / \n",
"10-Animal / 22-Nature / 588-Reptile / 763-Snake / \n",
"960-Badminton / \n",
"1-Vehicle / 27-Motorcycle / \n",
"14-Guitar / \n",
"9-Music video / \n",
"0-Games / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / \n",
"0-Games / 2140-Tony Hawk's / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 452-Watch / 1260-Smartwatch / \n",
"2610-Rosary / \n",
"9-Music video / \n",
"1-Vehicle / 25-Toy / 369-Resort / \n",
"13-Musical ensemble / \n",
"3-Concert / 36-Piano / 46-Choir / \n",
"6-Animation / 1303-Fullmetal Alchemist / \n",
"0-Games / 2-Video game / 1496-LittleBigPlanet 2 / 1672-LittleBigPlanet / \n",
"31-Disc jockey / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 387-Spider-Man / 3561-The Amazing Spider-Man / \n",
"1-Vehicle / 62-Train / 224-Rapid transit / 337-Advertising / 3348-BVE Trainsim / \n",
"208-Wood / 246-Furniture / \n",
"6-Animation / 15-Cartoon / \n",
"28-Fashion / 45-Cosmetics / 676-Handbag / \n",
"5-Dance / 31-Disc jockey / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 733-Washing / 1242-Car wash / \n",
"8-Football / 13-Musical ensemble / 203-Marching band / \n",
"3-Concert / 7-Musician / \n",
"25-Toy / 323-Toddler / 447-Sand / \n",
"12-Food / 476-Textile / \n",
"0-Games / 2-Video game / 731-Coin / \n",
"227-Water / 442-Bread / \n",
"36-Piano / 150-Violin / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"0-Games / 8-Football / 358-Rugby football / \n",
"7-Musician / 13-Musical ensemble / 21-Stadium / 49-School / 77-Arena / 79-American football / 203-Marching band / \n",
"6-Animation / 31-Disc jockey / \n",
"5-Dance / 49-School / 191-Cheerleading / 306-High school / \n",
"6-Animation / 2965-A Certain Magical Index / \n",
"369-Resort / \n",
"5-Dance / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / 1259-Fender Telecaster / \n",
"693-Sword / \n",
"7-Musician / 13-Musical ensemble / 21-Stadium / 49-School / 79-American football / 203-Marching band / 306-High school / \n",
"0-Games / 2-Video game / 43-Call of Duty / 194-Call of Duty: Modern Warfare 2 / \n",
"6-Animation / 15-Cartoon / \n",
"75-Wrestling / \n",
"5-Dance / 678-Latin dance / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 234-Ford / 473-Ford Mustang / 1877-Shelby Mustang / \n",
"0-Games / 211-RuneScape / \n",
"3-Concert / 720-Wine / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 1380-Kawasaki motorcycles / 4455-Kawasaki KLR650 / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"5-Dance / \n",
"6-Animation / 85-Combat / \n",
"1-Vehicle / 4-Car / 270-Sedan / 299-Volkswagen Passenger Cars / 342-Windows Media Video / 526-Volkswagen / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 363-Soil / 660-Rural area / \n",
"25-Toy / 99-Christmas / 200-Doll / 439-Barbie / 558-Santa Claus / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 169-Rallying / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 127-Xbox 360 / 2205-Bully / \n",
"3-Concert / 7-Musician / 24-String instrument / 150-Violin / 639-Fiddle / \n",
"1-Vehicle / 10-Animal / 17-Racing / 57-Race track / 80-Horse / 384-Horse racing / 1694-Harness racing / \n",
"14-Guitar / 110-Album / \n",
"0-Games / 79-American football / \n",
"49-School / \n",
"2233-Derek Shepherd / \n",
"155-Camera / 566-Digital camera / \n",
"66-Bollywood / \n",
"3-Concert / \n",
"0-Games / 60-Basketball / 168-Basketball moves / 255-Slam dunk / \n",
"164-Beach / 248-Hotel / 369-Resort / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 111-PlayStation 3 / 3832-Beyond: Two Souls / \n",
"10-Animal / 384-Horse racing / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / \n",
"330-Money / 731-Coin / 912-Silver / 4247-Bullion / \n",
"1-Vehicle / 50-Aircraft / 188-Airline / \n",
"246-Furniture / 462-Table / 920-Chair / \n",
"6-Animation / 121-Photography / 278-GoPro / \n",
"25-Toy / 126-The Walt Disney Company / 200-Doll / 354-Egg / 439-Barbie / 996-Princess / \n",
"20-Trailer / \n",
"2-Video game / 25-Toy / 140-LEGO / 229-Halo / 264-Star Wars / 852-Lego Star Wars / \n",
"12-Food / \n",
"9-Music video / \n",
"912-Silver / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"6-Animation / 61-Art / 97-Drawing / 148-Painting / 353-Wall / \n",
"229-Halo / 470-Halo 3 / \n",
"162-Medicine / \n",
"5-Dance / 115-Ballet / \n",
"0-Games / 6-Animation / 1878-SD Gundam Capsule Fighter / \n",
"125-Television / \n",
"1-Vehicle / 62-Train / \n",
"1-Vehicle / 4-Car / 74-Truck / 88-Machine / 249-Heavy equipment / 1093-Loader / 2445-Backhoe / \n",
"7-Musician / 13-Musical ensemble / 38-Orchestra / 230-Brass instrument / 1003-Tuba / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 403-Toyota / \n",
"1704-Antena 3 / \n",
"6-Animation / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / 1299-Shadow the Hedgehog / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 207-Exhaust system / 349-Chevrolet / 406-Chevrolet / 761-Chevrolet Camaro / \n",
"3-Concert / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / 397-Organ / 646-Pipe organ / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 139-Airport / 151-Landing / 175-Airliner / 472-Boeing 737 / \n",
"7-Musician / 13-Musical ensemble / 98-Festival / 203-Marching band / 250-Parade / 663-Street / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 152-Track / 428-Rail transport modelling / \n",
"1-Vehicle / 11-Motorsport / 124-Tractor / 156-Agriculture / 174-Farm / 249-Heavy equipment / 252-Tractor pulling / 363-Soil / 459-Plough / \n",
"0-Games / 2-Video game / 948-Far Cry / \n",
"641-Mask / \n",
"0-Games / 8-Football / 54-Highlight film / \n",
"12-Food / 215-Eating / 232-Cake / 600-Flour / \n",
"9-Music video / \n",
"5-Dance / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 3355-Bingo / \n",
"23-Mobile phone / 47-Personal computer / 131-Computer / \n",
"49-School / 306-High school / \n",
"1-Vehicle / 4-Car / 40-Road / 64-Transport / 74-Truck / 259-Fire / 371-Firefighter / 796-Fire engine / 1633-Fire station / \n",
"59-Winter sport / 84-Snow / 177-Skiing / 278-GoPro / 281-Snowboarding / 305-Ski / 501-Snowboard / \n",
"6-Animation / 15-Cartoon / 311-Street Fighter / 461-Street Fighter IV / 496-Super Street Fighter IV / 563-Ryu / 3936-Gouken / \n",
"0-Games / 2-Video game / 8-Football / 90-Sports game / 180-FIFA 15 / 480-FIFA 13 / \n",
"5-Dance / \n",
"15-Cartoon / 440-Lion / \n",
"22-Nature / 121-Photography / 155-Camera / 283-Rock / 629-Camera lens / 2248-Zoom lens / 3922-Wide-angle lens / \n",
"93-Comedy / \n",
"9-Music video / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"1-Vehicle / 4-Car / 298-Television advertisement / 683-Renault / \n",
"12-Food / 200-Doll / 1487-Reborn doll / \n",
"35-Minecraft / 242-Paper / 370-Origami / 385-Face / \n",
"0-Games / 2-Video game / 116-Pokémon / 186-Pokémon / 1204-Pokémon Ruby and Sapphire / 1665-Pokémon Emerald / 2236-Emerald / \n",
"313-Book / 2464-Bookcase / \n",
"12-Food / 26-Cooking / 32-Recipe / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / 40-Road / 105-Motorcycling / \n",
"28-Fashion / \n",
"75-Wrestling / \n",
"6-Animation / 25-Toy / 140-LEGO / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 517-Audi / \n",
"0-Games / 51-Strategy video game / 1718-Spore / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 76-Ball / 81-Athlete / \n",
"9-Music video / 31-Disc jockey / \n",
"1-Vehicle / 25-Toy / 62-Train / 1279-Plarail / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 158-Tire / 378-Drag racing / 1609-Auto Race / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 165-League of Legends / \n",
"9-Music video / \n",
"1-Vehicle / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 197-Trail / 220-Mountain bike / 284-Mountain biking / 645-Road bicycle / \n",
"163-Running / \n",
"0-Games / 21-Stadium / 54-Highlight film / 81-Athlete / 562-Pitcher / 946-Baseball park / \n",
"113-House / \n",
"205-Newscaster / 2642-CNBC / \n",
"59-Winter sport / 1829-Sledding / 4446-Bobsleigh / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 40-Road / 105-Motorcycling / 278-GoPro / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / 860-Christian Church / \n",
"0-Games / 2-Video game / 19-PC game / 23-Mobile phone / 35-Minecraft / 3253-Mobile game / \n",
"12-Food / 1424-Coupon / \n",
"47-Personal computer / 141-Microsoft Windows / 1043-Router / \n",
"66-Bollywood / \n",
"9-Music video / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 272-iPod / 331-iPod touch / \n",
"743-Edward Cullen / \n",
"116-Pokémon / \n",
"0-Games / 2-Video game / 1251-The Witcher / 2497-The Witcher 2: Assassins of Kings / \n",
"61-Art / 97-Drawing / 512-Printing / 1346-Wood carving / 2967-Printmaking / \n",
"10-Animal / 22-Nature / 132-Bird / \n",
"121-Photography / 293-Slide show / \n",
"3-Concert / 7-Musician / 38-Orchestra / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 135-Drifting / \n",
"1-Vehicle / 50-Aircraft / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 266-Helicopter / 392-Radio-controlled helicopter / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 135-Drifting / \n",
"1-Vehicle / 4-Car / 403-Toyota / 3219-Daihatsu / \n",
"83-Skateboarding / 121-Photography / 336-Scooter / 3866-Panasonic Lumix DMC-GH4 / \n",
"98-Festival / \n",
"0-Games / 2-Video game / 237-Final Fantasy / 1658-Dissidia Final Fantasy / 1857-Sephiroth / 2293-Cloud Strife / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 77-Arena / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 105-Motorcycling / 172-Motocross / \n",
"232-Cake / 1007-Portal / \n",
"8-Football / \n",
"0-Games / 2-Video game / 19-PC game / 506-Garry's Mod / 2290-Trouble in Terrorist Town / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / 172-Motocross / 491-Enduro / 1520-Minibike / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"0-Games / 25-Toy / 1154-Anpanman / \n",
"2-Video game / 1099-Euro Truck Simulator 2 / \n",
"0-Games / 35-Minecraft / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / \n",
"0-Games / 89-Comics / 285-Sasuke Uchiha / 486-M.U.G.E.N / \n",
"117-Boxing / \n",
"6-Animation / 25-Toy / 126-The Walt Disney Company / 200-Doll / \n",
"0-Games / 2-Video game / 20-Trailer / 111-PlayStation 3 / \n",
"6-Animation / 15-Cartoon / 1445-Donald Duck / \n",
"222-Surfing / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 299-Volkswagen Passenger Cars / 864-Speedometer / 1661-Volkswagen Gol / \n",
"260-Prayer / \n",
"10-Animal / 22-Nature / 280-Wildlife / \n",
"7-Musician / 150-Violin / \n",
"313-Book / \n",
"5-Dance / 616-The Bible / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 212-Highway / 473-Ford Mustang / \n",
"162-Medicine / \n",
"6-Animation / 15-Cartoon / 126-The Walt Disney Company / \n",
"98-Festival / \n",
"0-Games / 60-Basketball / \n",
"28-Fashion / \n",
"10-Animal / 22-Nature / 132-Bird / 217-Hunting / 585-Lawn / 1037-Grass / 4640-Decoy / \n",
"1-Vehicle / 84-Snow / 120-Winter / 459-Plough / 1145-Winter storm / 1548-Snowplow / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 102-Building / 129-Aviation / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 223-Weather / 254-Runway / 499-Cloud / \n",
"10-Animal / 12-Food / 48-Pet / 132-Bird / \n",
"9-Music video / \n",
"14-Guitar / 63-Acoustic guitar / 670-Banjo / \n",
"3-Concert / \n",
"325-Vampire / \n",
"75-Wrestling / \n",
"9-Music video / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 247-Stallion / 289-Dressage / 466-Gift / \n",
"209-University / \n",
"5-Dance / 6-Animation / 15-Cartoon / 1527-Pretty Cure / \n",
"3-Concert / 7-Musician / 16-Performance art / 38-Orchestra / \n",
"6-Animation / \n",
"3-Concert / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"130-Gymnastics / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"6-Animation / 2489-Sheriff Woody / \n",
"3-Concert / \n",
"3-Concert / 978-Love song / \n",
"708-Mermaid / \n",
"20-Trailer / 229-Halo / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 73-Ball / 76-Ball / 77-Arena / 81-Athlete / \n",
"10-Animal / 31-Disc jockey / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 428-Rail transport modelling / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 41-Sports car / 118-Grand Theft Auto V / \n",
"3-Concert / 5-Dance / 7-Musician / 16-Performance art / \n",
"1-Vehicle / 11-Motorsport / 50-Aircraft / 139-Airport / 157-Jet aircraft / 175-Airliner / 188-Airline / \n",
"3-Concert / 68-Lighting / \n",
"0-Games / 2-Video game / 20-Trailer / 176-PlayStation 4 / 474-Destiny / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 1469-X-Men / \n",
"3117-Match / \n",
"93-Comedy / \n",
"12-Food / 1158-Wheat / \n",
"5-Dance / 16-Performance art / \n",
"43-Call of Duty / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / \n",
"0-Games / 2-Video game / 20-Trailer / 43-Call of Duty / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / 3917-Eurogamer / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 118-Grand Theft Auto V / 201-Grand Theft Auto: San Andreas / \n",
"45-Cosmetics / 56-Hair / 106-Hairstyle / \n",
"1-Vehicle / 4-Car / 983-Lexus / \n",
"28-Fashion / 56-Hair / 106-Hairstyle / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 357-DVD / \n",
"28-Fashion / 45-Cosmetics / 153-Dress / 195-Mascara / 483-Costume / 757-Boot / 1586-Cowboy / \n",
"3-Concert / 16-Performance art / 31-Disc jockey / \n",
"10-Animal / 91-Fish / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 28-Fashion / \n",
"2-Video game / 171-Warcraft / \n",
"1-Vehicle / 87-Boat / 817-Bulldozer / \n",
"6-Animation / 9-Music video / 1934-Hunter × Hunter / \n",
"0-Games / 21-Stadium / 81-Athlete / 1111-Home run / \n",
"5-Dance / 49-School / 133-Nightclub / 253-Carnival / \n",
"45-Cosmetics / 1038-Hijab / \n",
"0-Games / 241-Fighting game / 311-Street Fighter / 486-M.U.G.E.N / 1331-Ken Masters / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 184-Vegetable / 274-Meat / 454-Sauce / 500-Rice / 671-Tomato / 3021-Tomato sauce / \n",
"5-Dance / \n",
"9-Music video / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"93-Comedy / \n",
"3-Concert / \n",
"0-Games / 2-Video game / \n",
"5-Dance / 1266-Death Note / 3661-Light Yagami / \n",
"0-Games / 2-Video game / 6-Animation / 2486-Scooby-Doo / \n",
"5-Dance / 98-Festival / 250-Parade / 253-Carnival / \n",
"1-Vehicle / 42-Fishing / 82-Airplane / 164-Beach / 223-Weather / 233-Ocean / 423-Coast / \n",
"6-Animation / 1206-Cinema 4D / \n",
"14-Guitar / \n",
"3-Concert / \n",
"9-Music video / 411-MapleStory / \n",
"0-Games / 88-Machine / 535-Slot machine / \n",
"0-Games / 2-Video game / 51-Strategy video game / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 19-PC game / 241-Fighting game / 486-M.U.G.E.N / \n",
"165-League of Legends / \n",
"0-Games / 159-World of Warcraft / \n",
"102-Building / 113-House / 416-Apartment / \n",
"1918-Comet / \n",
"260-Prayer / \n",
"0-Games / 51-Strategy video game / 165-League of Legends / 3259-Riven / \n",
"0-Games / 2-Video game / 19-PC game / 47-Personal computer / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 18-Outdoor recreation / 27-Motorcycle / 105-Motorcycling / 172-Motocross / 197-Trail / 491-Enduro / 1663-Two-stroke engine / 4308-Husaberg / \n",
"3-Concert / \n",
"287-Human swimming / 329-Swimming pool / 2605-Medley swimming / \n",
"18-Outdoor recreation / 267-Mountain / 283-Rock / 347-Climbing / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / 471-Wing / 719-Boeing 747 / 1340-Airbus A380 / \n",
"5-Dance / \n",
"38-Orchestra / \n",
"22-Nature / \n",
"6-Animation / 547-Bleach / \n",
"56-Hair / 106-Hairstyle / \n",
"359-Harry Potter (Literary Series) / 1436-Draco Malfoy / \n",
"3-Concert / 38-Orchestra / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 131-Computer / 189-iPad / 272-iPod / 331-iPod touch / \n",
"726-Clarinet / \n",
"10-Animal / 48-Pet / 132-Bird / \n",
"1-Vehicle / 4-Car / 12-Food / 475-Cheese / 1843-Ham / 2852-Burger King / \n",
"0-Games / 2-Video game / 141-Microsoft Windows / \n",
"9-Music video / 12-Food / 226-Dessert / \n",
"347-Climbing / 434-Ninja / \n",
"371-Firefighter / 415-Sewing / \n",
"0-Games / 256-Tennis / 544-Table tennis / \n",
"113-House / 707-Roof / \n",
"0-Games / 8-Football / 18-Outdoor recreation / 65-Kick / 73-Ball / 76-Ball / \n",
"12-Food / 26-Cooking / 2246-Snail / \n",
"9-Music video / \n",
"3-Concert / 21-Stadium / 68-Lighting / 365-Fireworks / \n",
"1-Vehicle / 10-Animal / 18-Outdoor recreation / 42-Fishing / 87-Boat / 91-Fish / 275-Diving / 288-Underwater / 481-Kayak / \n",
"78-Wedding / 263-Light / 1375-Laser lighting display / \n",
"3-Concert / 98-Festival / 594-Music festival / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"10-Animal / 537-Zoo / \n",
"96-Soldier / \n",
"12-Food / 640-Bottle / 1363-Candle / \n",
"0-Games / 2-Video game / 20-Trailer / 51-Strategy video game / 1570-Command & Conquer / 2990-Command & Conquer: Red Alert / 3430-Command & Conquer: Red Alert 3 / \n",
"6-Animation / 15-Cartoon / 1696-Raven / 2782-Starfire / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 335-Nintendo 3DS / 398-The Legend of Zelda / 2289-Super Mario Bros. 3 / 2589-Yoshi's Island / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 656-Metal Gear / 3284-Metal Gear Solid 2: Sons of Liberty / \n",
"3-Concert / 7-Musician / \n",
"1-Vehicle / 4-Car / 336-Scooter / 1448-Vespa / \n",
"506-Garry's Mod / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 17-Racing / 34-Action-adventure game / 111-PlayStation 3 / 118-Grand Theft Auto V / \n",
"28-Fashion / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 40-Road / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"170-Home improvement / 246-Furniture / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 950-Blu-ray disc / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 70-Driving / 94-Dashcam / 473-Ford Mustang / \n",
"8-Football / \n",
"3-Concert / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 428-Rail transport modelling / \n",
"243-Sonic the Hedgehog / 2236-Emerald / \n",
"6-Animation / 23-Mobile phone / 29-Smartphone / 101-Telephone / 1366-LG Optimus series / 4407-LG Optimus G / \n",
"66-Bollywood / \n",
"246-Furniture / 276-Room / 888-Living room / 1281-Couch / \n",
"14-Guitar / 465-Microphone / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 81-Athlete / \n",
"6-Animation / 950-Blu-ray disc / 1311-Amazon.com; Inc. / \n",
"0-Games / 942-Silkroad Online / \n",
"13-Musical ensemble / \n",
"3-Concert / \n",
"1-Vehicle / 74-Truck / 249-Heavy equipment / 433-Construction / 817-Bulldozer / \n",
"205-Newscaster / 766-President of the United States / \n",
"3-Concert / 219-Accordion / \n",
"219-Accordion / 655-Diatonic button accordion / \n",
"307-Hand / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 231-Supercar / 569-Lamborghini / \n",
"0-Games / 2-Video game / 685-Luigi / \n",
"689-Harmonica / 1023-Rose / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 74-Truck / 167-Four-wheel drive / 198-Off-road vehicle / 327-Mud / 565-Mud bogging / \n",
"15-Cartoon / 4598-Beavis / 4622-Butt-head / \n",
"1300-Earrings / 2814-Keychain / \n",
"75-Wrestling / 777-WWE 2K / \n",
"0-Games / 2-Video game / 89-Comics / 123-Naruto / 591-Naruto: Ultimate Ninja / \n",
"259-Fire / \n",
"0-Games / 8-Football / 60-Basketball / \n",
"10-Animal / 22-Nature / 86-Plant / 91-Fish / 258-Aquarium / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 70-Driving / 135-Drifting / 517-Audi / 982-V8 engine / \n",
"46-Choir / \n",
"10-Animal / 18-Outdoor recreation / 42-Fishing / 91-Fish / 288-Underwater / \n",
"12-Food / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"31-Disc jockey / \n",
"0-Games / 2-Video game / \n",
"13-Musical ensemble / \n",
"66-Bollywood / \n",
"0-Games / 2-Video game / 6-Animation / 20-Trailer / 123-Naruto / 185-Dragon Ball / 244-Goku / 285-Sasuke Uchiha / 396-One Piece / 682-PlayStation Vita / 1930-Rurouni Kenshin / 3232-Naruto Uzumaki / 3469-J-Stars Victory VS / 4078-Toriko / 4652-Weekly Shōnen Jump / \n",
"10-Animal / 80-Horse / 1434-Saddle / \n",
"14-Guitar / 46-Choir / 1231-Mandolin / \n",
"5-Dance / \n",
"9-Music video / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / \n",
"411-MapleStory / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 135-Drifting / 169-Rallying / \n",
"3-Concert / 31-Disc jockey / 133-Nightclub / \n",
"242-Paper / 476-Textile / 1057-Embroidery / \n",
"0-Games / 8-Football / \n",
"9-Music video / \n",
"76-Ball / 256-Tennis / 974-Racket / 1337-Serve / \n",
"3-Concert / 342-Windows Media Video / \n",
"402-Jewellery / 829-Bead / 859-Beadwork / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 41-Sports car / 207-Exhaust system / 234-Ford / 473-Ford Mustang / 647-Muscle car / 1361-Ford GT / 1877-Shelby Mustang / 3755-AC Cobra / \n",
"458-Gold / 768-Diamond / \n",
"14-Guitar / \n",
"36-Piano / 53-Keyboard / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 243-Sonic the Hedgehog / 3366-Amnesia: The Dark Descent / \n",
"9-Music video / \n",
"3-Concert / \n",
"9-Music video / \n",
"18-Outdoor recreation / 59-Winter sport / 84-Snow / 177-Skiing / 281-Snowboarding / 305-Ski / 501-Snowboard / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"3-Concert / 14-Guitar / 24-String instrument / \n",
"483-Costume / 2556-Halloween costume / \n",
"0-Games / 8-Football / 21-Stadium / 73-Ball / 77-Arena / 90-Sports game / \n",
"3-Concert / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 155-Camera / 278-GoPro / \n",
"5-Dance / \n",
"5-Dance / \n",
"5-Dance / 16-Performance art / \n",
"5-Dance / 16-Performance art / 115-Ballet / 678-Latin dance / \n",
"209-University / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / \n",
"2-Video game / 37-Gadget / 55-Video game console / 292-Handheld game console / 362-PlayStation Portable / \n",
"0-Games / 2-Video game / 6-Animation / 20-Trailer / 55-Video game console / 111-PlayStation 3 / 362-PlayStation Portable / \n",
"110-Album / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / 606-String / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 25-Toy / 92-Radio-controlled model / 202-Radio-controlled car / \n",
"45-Cosmetics / 56-Hair / 385-Face / 2108-Hair removal / \n",
"0-Games / 60-Basketball / 76-Ball / \n",
"1-Vehicle / 27-Motorcycle / \n",
"147-River / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 178-Samsung Galaxy / 932-Microsoft Lumia / 4611-Nokia Lumia 800 / \n",
"9-Music video / \n",
"938-Steel / 2221-Deck / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 1396-Lara Croft / 3892-Tomb Raider / \n",
"122-Weight training / 204-Gym / \n",
"1836-Fly tying / \n",
"1680-Seafight / \n",
"5-Dance / 361-Flamenco / \n",
"8-Football / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 135-Drifting / 1887-Tricycle / 3614-Drift trikes / \n",
"0-Games / 49-School / 54-Highlight film / 60-Basketball / 168-Basketball moves / \n",
"2337-Passport / \n",
"912-Silver / \n",
"7-Musician / 14-Guitar / \n",
"3-Concert / 7-Musician / 14-Guitar / 63-Acoustic guitar / \n",
"28-Fashion / 68-Lighting / 121-Photography / 155-Camera / 263-Light / \n",
"215-Eating / \n",
"45-Cosmetics / 56-Hair / 193-Eye liner / 196-Eye / 218-Eyelash / \n",
"31-Disc jockey / \n",
"37-Gadget / 121-Photography / 155-Camera / 629-Camera lens / 1855-Video camera / \n",
"12-Food / 45-Cosmetics / 173-Eye shadow / 195-Mascara / 215-Eating / 218-Eyelash / 235-Lipstick / \n",
"121-Photography / 155-Camera / \n",
"5-Dance / 1199-Kinect / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"5-Dance / \n",
"23-Mobile phone / 29-Smartphone / \n",
"9-Music video / \n",
"6-Animation / 15-Cartoon / 750-Sailor Moon / 1342-Sailor Moon / 1551-The Phantom of the Opera / \n",
"6-Animation / 10-Animal / 15-Cartoon / 166-Cat / 1209-Bean / \n",
"282-Skin / \n",
"3-Concert / 5-Dance / 16-Performance art / 93-Comedy / \n",
"3-Concert / 14-Guitar / 63-Acoustic guitar / \n",
"1057-Embroidery / \n",
"10-Animal / 17-Racing / 384-Horse racing / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 105-Motorcycling / 172-Motocross / \n",
"512-Printing / 525-Printer / 1751-Image scanner / 2634-Inkjet printing / 3195-Multifunction printer / \n",
"3-Concert / 5-Dance / 13-Musical ensemble / 14-Guitar / 16-Performance art / 39-Drums / 44-Drums / 100-Electric guitar / 376-Flute / \n",
"0-Games / 51-Strategy video game / 353-Wall / 373-Clash of Clans / \n",
"121-Photography / \n",
"42-Fishing / 573-Paragliding / \n",
"18-Outdoor recreation / 164-Beach / 222-Surfing / \n",
"5-Dance / 95-Talent show / 191-Cheerleading / \n",
"6-Animation / 15-Cartoon / \n",
"250-Parade / 253-Carnival / \n",
"205-Newscaster / \n",
"9-Music video / \n",
"10-Animal / 540-Insect / 775-Bee / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / 49-School / \n",
"6-Animation / 15-Cartoon / 412-Monster / 503-Monster High / \n",
"3-Concert / 8-Football / \n",
"9-Music video / 78-Wedding / 190-Bride / \n",
"1-Vehicle / 64-Transport / 205-Newscaster / 286-Bus / \n",
"1095-Shiva / \n",
"9-Music video / \n",
"13-Musical ensemble / \n",
"10-Animal / 588-Reptile / 1444-Lizard / 3576-Iguana / \n",
"15-Cartoon / 61-Art / 2550-EarthBound / \n",
"20-Trailer / \n",
"60-Basketball / 1399-Streetball / 3652-3x3 / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 184-Vegetable / 542-Potato / 1355-Vegetarian food / 4700-Shashlik / \n",
"1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 70-Driving / 135-Drifting / 231-Supercar / 705-Forza Motorsport / 2105-Forza Horizon / 2106-Forza Horizon 2 / 2210-McLaren / 3748-McLaren 12C / \n",
"39-Drums / 724-Rock Band / 1703-Rock Band / \n",
"209-University / \n",
"1459-Magazine / 2493-The Hobbit / \n",
"93-Comedy / \n",
"3342-The Mad Hatter / \n",
"0-Games / 2-Video game / 19-PC game / 4140-Quake Live / \n",
"122-Weight training / 204-Gym / 214-Muscle / 1430-Pull-up / \n",
"1-Vehicle / 249-Heavy equipment / 363-Soil / 1093-Loader / \n",
"9-Music video / \n",
"3-Concert / 16-Performance art / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / \n",
"28-Fashion / 153-Dress / 390-Model / 495-Runway / \n",
"56-Hair / 106-Hairstyle / 340-Afro-textured hair / 954-Dreadlocks / \n",
"3-Concert / \n",
"5-Dance / 6-Animation / 126-The Walt Disney Company / \n",
"23-Mobile phone / 29-Smartphone / 380-Assassin's Creed / 553-Sony Xperia / 4440-Sony Ericsson Xperia X8 / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 270-Sedan / 403-Toyota / 2104-Toyota Camry / \n",
"3-Concert / 16-Performance art / \n",
"7-Musician / 14-Guitar / 24-String instrument / \n",
"3-Concert / 16-Performance art / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / 40-Road / 64-Transport / 94-Dashcam / \n",
"0-Games / 535-Slot machine / \n",
"0-Games / 8-Football / 79-American football / \n",
"1-Vehicle / 4-Car / 74-Truck / 154-Wheel / 167-Four-wheel drive / 234-Ford / 375-Pickup truck / 515-Dodge / 1437-Ram Trucks / 1651-Dodge Ram / \n",
"3-Concert / 46-Choir / \n",
"45-Cosmetics / 173-Eye shadow / 238-Nail / 304-Nail polish / \n",
"0-Games / 686-Diablo III / 872-Diablo / \n",
"6-Animation / 15-Cartoon / 3689-Woody Woodpecker / \n",
"5-Dance / 115-Ballet / \n",
"1026-Foreign Exchange Market / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 782-Super Smash Bros. Melee / 1166-Dark Souls II / \n",
"458-Gold / 466-Gift / \n",
"72-iPhone / 508-iPhone 5 / \n",
"0-Games / 8-Football / 73-Ball / 90-Sports game / 111-PlayStation 3 / 453-Pro Evolution Soccer / \n",
"14-Guitar / \n",
"3-Concert / 7-Musician / \n",
"6-Animation / 477-Transformers / 990-Optimus Prime / \n",
"1-Vehicle / 4-Car / 40-Road / 74-Truck / 575-Traffic / \n",
"102-Building / 113-House / 170-Home improvement / 276-Room / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 151-Landing / 175-Airliner / 719-Boeing 747 / 923-Boeing 777 / \n",
"0-Games / 60-Basketball / 168-Basketball moves / 255-Slam dunk / \n",
"0-Games / 2-Video game / 903-Mass Effect / 1655-Mass Effect 2 / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / \n",
"0-Games / 47-Personal computer / 131-Computer / 1293-RollerCoaster Tycoon 3 / \n",
"1-Vehicle / 4-Car / 9-Music video / 531-Hatchback / 988-Ibiza / 3990-SEAT Ibiza / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / \n",
"83-Skateboarding / \n",
"28-Fashion / \n",
"0-Games / 201-Grand Theft Auto: San Andreas / 1612-Multi Theft Auto / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"10-Animal / 16-Performance art / 61-Art / 636-Rabbit / 981-Sculpture / \n",
"376-Flute / \n",
"156-Agriculture / 174-Farm / 715-Farming Simulator / 3260-Barley / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / 401-Cello / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 388-Fighter aircraft / 758-Air force / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / \n",
"12-Food / 482-Asphalt / \n",
"0-Games / 35-Minecraft / \n",
"0-Games / 51-Strategy video game / \n",
"3-Concert / 13-Musical ensemble / 46-Choir / \n",
"783-Zee TV / \n",
"0-Games / 2-Video game / 37-Gadget / 55-Video game console / 292-Handheld game console / 362-PlayStation Portable / \n",
"0-Games / 8-Football / 73-Ball / 90-Sports game / 180-FIFA 15 / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 506-Garry's Mod / \n",
"6-Animation / 10-Animal / 15-Cartoon / 538-Illustration / 2697-Zebra / \n",
"20-Trailer / \n",
"489-Board game / 799-Chess / \n",
"9-Music video / \n",
"45-Cosmetics / 56-Hair / 871-Hair coloring / \n",
"10-Animal / 48-Pet / 132-Bird / 3121-Homing pigeon / \n",
"0-Games / 33-Weapon / 664-CrossFire / \n",
"1-Vehicle / 4-Car / 349-Chevrolet / 406-Chevrolet / 647-Muscle car / 761-Chevrolet Camaro / \n",
"163-Running / 1013-Walking / \n",
"3-Concert / 7-Musician / \n",
"1550-Newspaper / \n",
"1-Vehicle / 87-Boat / 949-Rowing / \n",
"7-Musician / 14-Guitar / 24-String instrument / 36-Piano / 53-Keyboard / 100-Electric guitar / 1103-Gibson Les Paul / \n",
"3-Concert / \n",
"5-Dance / 16-Performance art / \n",
"6-Animation / 15-Cartoon / 97-Drawing / 183-Manga / 1987-Reborn! / \n",
"3-Concert / 5-Dance / 16-Performance art / 361-Flamenco / \n",
"10-Animal / 48-Pet / 280-Wildlife / 706-Deer / \n",
"10-Animal / 48-Pet / 71-Dog / 2350-Flying disc / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / 556-Skateboarding trick / \n",
"0-Games / 2-Video game / 83-Skateboarding / 2140-Tony Hawk's / \n",
"12-Food / 26-Cooking / 52-Dish / 58-Cuisine / \n",
"0-Games / 2-Video game / 89-Comics / 2368-Marvel Universe / \n",
"96-Soldier / 1573-United States Army / \n",
"0-Games / 2-Video game / 19-PC game / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 41-Sports car / 207-Exhaust system / 403-Toyota / 3416-Toyota MR2 / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 291-Rouge / \n",
"0-Games / 319-Robot / \n",
"23-Mobile phone / \n",
"0-Games / 2-Video game / 116-Pokémon / 186-Pokémon / 1387-Pokémon Battle Revolution / \n",
"17-Racing / 23-Mobile phone / 29-Smartphone / 37-Gadget / 67-Cycling / 69-Bicycle / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 73-Ball / 76-Ball / 81-Athlete / \n",
"9-Music video / \n",
"116-Pokémon / 1104-Ash Ketchum / 3347-Misty / \n",
"5-Dance / 49-School / 95-Talent show / 191-Cheerleading / \n",
"16-Performance art / 49-School / \n",
"9-Music video / 905-Flash Video / \n",
"3-Concert / 7-Musician / 465-Microphone / \n",
"7-Musician / 46-Choir / \n",
"20-Trailer / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / 224-Rapid transit / \n",
"262-Ice / 312-Ice rink / 833-Ice skate / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / 842-Harp / \n",
"75-Wrestling / \n",
"0-Games / 8-Football / 21-Stadium / 77-Arena / \n",
"5-Dance / \n",
"1-Vehicle / 4-Car / 6-Animation / 25-Toy / 140-LEGO / 940-Clay animation / 1079-Lego City / \n",
"9-Music video / 938-Steel / \n",
"754-Gramophone record / \n",
"14-Guitar / 100-Electric guitar / \n",
"1-Vehicle / 4-Car / 864-Speedometer / 3229-Opel Kadett / \n",
"12-Food / 1162-Dietary supplement / 3498-Whey protein / \n",
"20-Trailer / \n",
"222-Surfing / \n",
"313-Book / 748-Iron Man / 855-Hulk / 1803-Ultron / 3109-Clint Barton / \n",
"0-Games / 1-Vehicle / 2-Video game / 6-Animation / 271-Tank / 409-World of Tanks / \n",
"28-Fashion / 206-Shoe / 333-Nike; Inc. / 368-Sneakers / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 487-Samsung Electronics / \n",
"3-Concert / 7-Musician / 38-Orchestra / \n",
"3-Concert / \n",
"3-Concert / 68-Lighting / 263-Light / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / \n",
"163-Running / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 194-Call of Duty: Modern Warfare 2 / 302-Call of Duty 4: Modern Warfare / 1425-Gears of War 2 / \n",
"60-Basketball / 3905-Slam Dunk / \n",
"15-Cartoon / 111-PlayStation 3 / 877-Hatsune Miku: Project DIVA / \n",
"0-Games / 2-Video game / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 188-Airline / 254-Runway / \n",
"12-Food / \n",
"1-Vehicle / 4-Car / 74-Truck / 167-Four-wheel drive / 234-Ford / 375-Pickup truck / \n",
"8-Football / 21-Stadium / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 2-Video game / 20-Trailer / 43-Call of Duty / 138-Call of Duty: Black Ops II / 394-Map / \n",
"0-Games / 2-Video game / 75-Wrestling / 85-Combat / 90-Sports game / 111-PlayStation 3 / 117-Boxing / 127-Xbox 360 / 181-Xbox / 700-Hard disk drive / 3907-UFC Undisputed 3 / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 432-Kingdom Hearts / 665-Kingdom Hearts / 836-Sora / 941-Kingdom Hearts II / 3299-Kingdom Hearts: Chain of Memories / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"5-Dance / 16-Performance art / 980-Solo dance / \n",
"0-Games / 2-Video game / 89-Comics / 185-Dragon Ball / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 179-Kitchen / 210-Cookware and bakeware / 239-Roasting / 533-Oil / 662-Pizza / 2483-Deep frying / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 297-Grand Theft Auto IV / \n",
"5-Dance / 16-Performance art / 123-Naruto / \n",
"6-Animation / 102-Building / 433-Construction / 666-Concrete / \n",
"415-Sewing / 571-Bear / 1152-Plush / 1212-Stuffed toy / 2329-Felt / \n",
"5-Dance / 31-Disc jockey / \n",
"18-Outdoor recreation / 42-Fishing / 395-Camping / 914-Tent / \n",
"444-Family / \n",
"1-Vehicle / 11-Motorsport / 124-Tractor / 156-Agriculture / 252-Tractor pulling / 459-Plough / \n",
"455-Unidentified flying object / \n",
"1-Vehicle / 7-Musician / 13-Musical ensemble / 49-School / 54-Highlight film / 79-American football / 203-Marching band / 306-High school / \n",
"59-Winter sport / 84-Snow / 177-Skiing / 305-Ski / 789-Alpine skiing / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 704-Call of Duty: World at War / 1248-Call of Duty: Zombies / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"0-Games / 2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / \n",
"59-Winter sport / 83-Skateboarding / 108-Skateboard / 112-Ice skating / 222-Surfing / \n",
"673-Saw / 4161-Harbor Freight Tools / \n",
"3002-Essential oil / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 81-Athlete / 358-Rugby football / \n",
"5-Dance / 49-School / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 57-Race track / 169-Rallying / 2518-Dirt 3 / \n",
"455-Unidentified flying object / \n",
"1-Vehicle / 25-Toy / 27-Motorcycle / 140-LEGO / 1478-Lego Ninjago / 1925-Skull / \n",
"1-Vehicle / 4-Car / 6-Animation / \n",
"1-Vehicle / 6-Animation / 25-Toy / 50-Aircraft / 82-Airplane / 139-Airport / 140-LEGO / 1079-Lego City / 1302-The Lego Group / \n",
"20-Trailer / 229-Halo / 866-Halo 4 / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 105-Motorcycling / \n",
"8-Football / 21-Stadium / \n",
"78-Wedding / \n",
"45-Cosmetics / 56-Hair / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 235-Lipstick / 282-Skin / 291-Rouge / 385-Face / 436-Concealer / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 216-Call of Duty: Modern Warfare 3 / \n",
"0-Games / 2-Video game / 19-PC game / 192-Battlefield / 374-Battlefield 3 / \n",
"0-Games / 1-Vehicle / 11-Motorsport / 17-Racing / 57-Race track / \n",
"10-Animal / 48-Pet / 71-Dog / \n",
"12-Food / 26-Cooking / 52-Dish / 58-Cuisine / 215-Eating / 771-Noodle / 1438-Street food / 2178-Chinese food / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 569-Lamborghini / \n",
"0-Games / 4685-Shot put / \n",
"2642-CNBC / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 1084-Bulldog / 2438-French Bulldog / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 96-Soldier / 638-Gears of War / 773-Gears of War / 1425-Gears of War 2 / \n",
"0-Games / 35-Minecraft / \n",
"10-Animal / 91-Fish / 258-Aquarium / \n",
"3-Concert / 5-Dance / 78-Wedding / \n",
"273-News program / \n",
"1-Vehicle / 4-Car / 345-Loudspeaker / 484-Subwoofer / \n",
"45-Cosmetics / 196-Eye / \n",
"3-Concert / \n",
"5-Dance / \n",
"0-Games / 1-Vehicle / 2-Video game / 34-Action-adventure game / 297-Grand Theft Auto IV / 658-Grand Theft Auto: The Lost and Damned / \n",
"1107-United States Navy / \n",
"9-Music video / 843-The Doctor / \n",
"205-Newscaster / 307-Hand / \n",
"9-Music video / 121-Photography / 153-Dress / \n",
"242-Paper / 370-Origami / \n",
"9-Music video / \n",
"45-Cosmetics / 56-Hair / 106-Hairstyle / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 2834-Huawei Ascend / \n",
"110-Album / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 101-Telephone / 2191-Text messaging / \n",
"1-Vehicle / 27-Motorcycle / \n",
"8-Football / 21-Stadium / \n",
"205-Newscaster / 354-Egg / 500-Rice / \n",
"0-Games / 60-Basketball / 168-Basketball moves / 255-Slam dunk / \n",
"10-Animal / 12-Food / 26-Cooking / 32-Recipe / 91-Fish / 274-Meat / 893-Fish as food / \n",
"10-Animal / 91-Fish / 258-Aquarium / 716-Coral / 1051-Reef aquarium / 1368-Gallon / \n",
"79-American football / \n",
"60-Basketball / 3291-Basketball Coach / \n",
"22-Nature / 275-Diving / 288-Underwater / 381-Underwater diving / 407-Scuba diving / 425-City / \n",
"75-Wrestling / 1042-Jumbotron / \n",
"59-Winter sport / 177-Skiing / 789-Alpine skiing / 1764-Ski jumping / \n",
"78-Wedding / \n",
"8-Football / 350-Circus / 788-Juggling / \n",
"222-Surfing / 672-Bodyboarding / \n",
"46-Choir / 1501-Military parade / \n",
"5-Dance / \n",
"12-Food / 26-Cooking / 533-Oil / \n",
"0-Games / 1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 1817-Initial D Arcade Stage / \n",
"66-Bollywood / \n",
"9-Music video / \n",
"45-Cosmetics / 919-Hair conditioner / \n",
"8-Football / \n",
"10-Animal / 71-Dog / \n",
"76-Ball / 3264-Golf ball / \n",
"5-Dance / 16-Performance art / 1289-Caporales / \n",
"5-Dance / 16-Performance art / 115-Ballet / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 179-Kitchen / 210-Cookware and bakeware / 239-Roasting / 274-Meat / 1186-Seafood / \n",
"5-Dance / 6-Animation / 15-Cartoon / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / \n",
"1-Vehicle / 4-Car / 270-Sedan / 3134-Mazda6 / \n",
"12-Food / 136-Gardening / 666-Concrete / \n",
"20-Trailer / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 38-Orchestra / 39-Drums / \n",
"10-Animal / 14-Guitar / 48-Pet / 71-Dog / 2271-Boxer / \n",
"67-Cycling / 69-Bicycle / 83-Skateboarding / 221-Skatepark / 377-BMX bike / \n",
"42-Fishing / 217-Hunting / \n",
"3-Concert / 68-Lighting / 258-Aquarium / \n",
"9-Music video / \n",
"59-Winter sport / 84-Snow / 177-Skiing / 305-Ski / 789-Alpine skiing / \n",
"5-Dance / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / 49-School / \n",
"28-Fashion / \n",
"5-Dance / 95-Talent show / \n",
"10-Animal / 709-Bull / \n",
"6-Animation / \n",
"0-Games / 2086-Dominoes / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 76-Ball / 81-Athlete / \n",
"0-Games / 2-Video game / 19-PC game / 3362-WildStar / 4655-The Secret World / \n",
"28-Fashion / 78-Wedding / 153-Dress / 190-Bride / 421-Gown / 590-Wedding dress / 607-Lace / \n",
"5-Dance / 2341-Fine art / \n",
"259-Fire / 371-Firefighter / \n",
"1-Vehicle / 4-Car / 270-Sedan / 517-Audi / 1587-Audi Quattro / 2306-Audi A6 / \n",
"75-Wrestling / 85-Combat / 287-Human swimming / \n",
"6-Animation / 15-Cartoon / 61-Art / 89-Comics / 183-Manga / 1696-Raven / 2782-Starfire / 2945-Fan art / \n",
"6-Animation / 15-Cartoon / 1866-Katara / 3734-Aang / \n",
"12-Food / 301-Meal / \n",
"35-Minecraft / \n",
"5-Dance / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 39-Drums / \n",
"9-Music video / \n",
"61-Art / 479-Graffiti / \n",
"0-Games / 8-Football / 21-Stadium / 73-Ball / 77-Arena / 90-Sports game / 180-FIFA 15 / \n",
"0-Games / 2-Video game / 19-PC game / 166-Cat / 850-Left 4 Dead / 963-Left 4 Dead 2 / \n",
"1-Vehicle / 4-Car / 40-Road / 74-Truck / 167-Four-wheel drive / 198-Off-road vehicle / 277-Sport utility vehicle / 375-Pickup truck / 901-Mitsubishi / \n",
"0-Games / 2-Video game / 635-Achievement / 831-Plants vs. Zombies / \n",
"8-Football / 164-Beach / \n",
"12-Food / 475-Cheese / \n",
"0-Games / 2-Video game / 35-Minecraft / 47-Personal computer / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"8-Football / \n",
"6-Animation / 15-Cartoon / 89-Comics / 801-Inuyasha / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / \n",
"56-Hair / 106-Hairstyle / 340-Afro-textured hair / 405-Braid / 857-Afro / 1812-Hair twists / 2351-Moisturizer / 2356-Comb / \n",
"1-Vehicle / 11-Motorsport / 67-Cycling / 69-Bicycle / 220-Mountain bike / \n",
"12-Food / 26-Cooking / 32-Recipe / 226-Dessert / 232-Cake / 437-Cream / 475-Cheese / 560-Sugar / 759-Icing / 841-Butter / 1853-Whipped cream / \n",
"31-Disc jockey / \n",
"6-Animation / 15-Cartoon / 3734-Aang / \n",
"1076-Wakeboarding / \n",
"10-Animal / 751-Desert / 2410-Camel / \n",
"12-Food / 52-Dish / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 89-Comics / 185-Dragon Ball / 244-Goku / 869-Dragon Ball Z: Budokai Tenkaichi / 876-Freeza / \n",
"0-Games / 2-Video game / 19-PC game / 1071-Hearthstone: Heroes of Warcraft / \n",
"5-Dance / 191-Cheerleading / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 47-Personal computer / 145-Tablet computer / 178-Samsung Galaxy / 1287-Samsung Galaxy Tab series / 3863-Temple Run / \n",
"12-Food / \n",
"125-Television / 609-Tattoo / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"0-Games / 2-Video game / 2712-Just Cause / \n",
"102-Building / 113-House / 339-Architecture / \n",
"9-Music video / \n",
"0-Games / 8-Football / 79-American football / 81-Athlete / \n",
"68-Lighting / 3803-Jaguar / \n",
"6-Animation / \n",
"5-Dance / \n",
"122-Weight training / \n",
"0-Games / 1680-Seafight / \n",
"1684-Hillsong Church / \n",
"0-Games / 2-Video game / 320-Super Smash Bros. / 782-Super Smash Bros. Melee / 2085-Falco Lombardi / 2165-Ganon / \n",
"0-Games / 6-Animation / 141-Microsoft Windows / \n",
"3-Concert / 7-Musician / 3489-Rock and Roll Hall of Fame / \n",
"0-Games / 1-Vehicle / 2-Video game / 11-Motorsport / 456-Go-kart / 518-Mario Kart / 1831-Mario Kart 8 / 2901-Mario Kart 64 / 3095-Mario Kart DS / 3346-Super Mario Kart / \n",
"1-Vehicle / 345-Loudspeaker / \n",
"1-Vehicle / 62-Train / 102-Building / \n",
"359-Harry Potter (Literary Series) / 1436-Draco Malfoy / 1856-Hogwarts School of Witchcraft and Wizardry / 1912-Lord Voldemort / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 36-Piano / 46-Choir / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / \n",
"1-Vehicle / 4-Car / 245-BMW / \n",
"0-Games / 577-Penguin / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 70-Driving / 1537-Audi R8 / 2949-V10 engine / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 84-Snow / 172-Motocross / 278-GoPro / 4014-Yamaha YZ250 / 4436-Yamaha YZ450F / 4536-Kawasaki KX250F / \n",
"0-Games / 8-Football / \n",
"0-Games / 2-Video game / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 1036-Moped / \n",
"0-Games / 2-Video game / 23-Mobile phone / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 30-Drummer / 128-Cymbal / \n",
"93-Comedy / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 24-String instrument / \n",
"1-Vehicle / 87-Boat / 308-Ship / 579-Yacht / \n",
"9-Music video / \n",
"10-Animal / 18-Outdoor recreation / 42-Fishing / 91-Fish / 478-Fishing rod / \n",
"25-Toy / 140-LEGO / \n",
"12-Food / 86-Plant / 225-Lake / \n",
"5-Dance / 31-Disc jockey / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 239-Roasting / 500-Rice / 572-Indian cuisine / 727-Curry / 808-Breakfast / \n",
"6-Animation / \n",
"9-Music video / \n",
"49-School / 1640-Library / \n",
"102-Building / 259-Fire / \n",
"248-Hotel / \n",
"66-Bollywood / \n",
"925-Brain / \n",
"341-Sketch comedy / \n",
"0-Games / 2-Video game / 19-PC game / 192-Battlefield / 383-Battlefield 4 / \n",
"75-Wrestling / 1441-Mattel / \n",
"720-Wine / \n",
"2-Video game / 55-Video game console / 111-PlayStation 3 / 127-Xbox 360 / 199-Wii / 317-PlayStation / \n",
"3218-Dragonica / \n",
"2-Video game / 9-Music video / 229-Halo / 644-Halo: Reach / 866-Halo 4 / 2240-Halo 5: Guardians / 2796-Halo: The Master Chief Collection / \n",
"250-Parade / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"467-Chipmunk / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 19-PC game / 159-World of Warcraft / 171-Warcraft / \n",
"0-Games / 19-PC game / 211-RuneScape / \n",
"0-Games / 2-Video game / 51-Strategy video game / \n",
"144-Amusement park / 321-Roller coaster / 334-Amusement ride / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 96-Soldier / 1394-Alliance of Valiant Arms / \n",
"28-Fashion / 206-Shoe / 757-Boot / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"33-Weapon / 693-Sword / \n",
"3-Concert / 5-Dance / \n",
"3-Concert / \n",
"23-Mobile phone / 29-Smartphone / 802-Google Nexus / \n",
"325-Vampire / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 135-Drifting / 169-Rallying / 1041-Subaru Impreza / 1286-Subaru / \n",
"9-Music video / \n",
"516-Headphones / 832-Headset / \n",
"2-Video game / 83-Skateboarding / 90-Sports game / 2140-Tony Hawk's / \n",
"7-Musician / 13-Musical ensemble / 38-Orchestra / 219-Accordion / \n",
"1-Vehicle / 88-Machine / 512-Printing / \n",
"28-Fashion / 56-Hair / 106-Hairstyle / \n",
"3-Concert / 5-Dance / \n",
"3895-Curling / \n",
"1-Vehicle / 4-Car / 70-Driving / 270-Sedan / 332-Mercedes-Benz / 2365-Mercedes-Benz S-Class / \n",
"89-Comics / 269-Comic book / 387-Spider-Man / \n",
"350-Circus / \n",
"0-Games / 2-Video game / 127-Xbox 360 / 386-Xbox / 948-Far Cry / 1593-Digital video recorder / \n",
"0-Games / 2-Video game / 6-Animation / 19-PC game / 1343-Super Robot Wars / 3128-Super Robot / \n",
"1325-Gel / 2731-Aloe vera / 3770-Toothpaste / \n",
"1-Vehicle / 4-Car / \n",
"273-News program / \n",
"68-Lighting / 133-Nightclub / \n",
"9-Music video / \n",
"5-Dance / 16-Performance art / 153-Dress / 762-Angel / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / \n",
"298-Television advertisement / 337-Advertising / \n",
"1-Vehicle / 2903-Umbrella / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 178-Samsung Galaxy / 1207-Samsung Galaxy S5 / \n",
"1-Vehicle / 12-Food / 2937-Banknote / \n",
"9-Music video / \n",
"0-Games / 379-Touhou Project / 3849-Perfect Cherry Blossom / \n",
"3-Concert / 7-Musician / \n",
"28-Fashion / 495-Runway / \n",
"5-Dance / 28-Fashion / \n",
"102-Building / 113-House / 170-Home improvement / \n",
"3-Concert / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 2413-Union Pacific Railroad / \n",
"0-Games / 95-Talent show / 191-Cheerleading / \n",
"38-Orchestra / \n",
"5-Dance / 153-Dress / 430-Ballroom dance / 678-Latin dance / \n",
"571-Bear / 2134-Teddy bear / \n",
"12-Food / 1022-Chili pepper / \n",
"1816-Fire Emblem / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 36-Piano / 46-Choir / \n",
"452-Watch / 627-Clock / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / \n",
"1-Vehicle / 62-Train / 259-Fire / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 105-Motorcycling / 172-Motocross / 197-Trail / 213-Forest / 363-Soil / \n",
"0-Games / 2-Video game / 55-Video game console / 237-Final Fantasy / 335-Nintendo 3DS / \n",
"3-Concert / \n",
"121-Photography / 155-Camera / \n",
"0-Games / 211-RuneScape / \n",
"93-Comedy / 264-Star Wars / \n",
"0-Games / 2-Video game / 55-Video game console / 199-Wii / 362-PlayStation Portable / \n",
"6-Animation / 15-Cartoon / 307-Hand / 733-Washing / \n",
"28-Fashion / 45-Cosmetics / 56-Hair / \n",
"93-Comedy / 293-Slide show / \n",
"0-Games / 1-Vehicle / 2-Video game / 124-Tractor / 174-Farm / 715-Farming Simulator / \n",
"0-Games / 2-Video game / \n",
"133-Nightclub / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 94-Dashcam / 212-Highway / 668-Volkswagen Golf / \n",
"3-Concert / 7-Musician / \n",
"9-Music video / \n",
"3-Concert / \n",
"341-Sketch comedy / 364-Sitcom / \n",
"5-Dance / \n",
"1-Vehicle / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 197-Trail / 220-Mountain bike / 284-Mountain biking / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 40-Road / 336-Scooter / 3685-Pulley / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 210-Cookware and bakeware / 727-Curry / 1024-Culinary art / 1209-Bean / 3100-Chickpea / \n",
"10-Animal / 18-Outdoor recreation / 42-Fishing / 91-Fish / 268-Recreational fishing / 601-Shark / \n",
"345-Loudspeaker / 484-Subwoofer / \n",
"3-Concert / 5-Dance / 16-Performance art / 115-Ballet / \n",
"6-Animation / 2951-Black Rock Shooter / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / 318-Church / \n",
"22-Nature / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 207-Exhaust system / 234-Ford / 473-Ford Mustang / 647-Muscle car / 650-Convertible / 1361-Ford GT / \n",
"0-Games / 2-Video game / 903-Mass Effect / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 952-Chevrolet Corvette / \n",
"20-Trailer / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 490-Rain / 645-Road bicycle / 1887-Tricycle / \n",
"72-iPhone / \n",
"31-Disc jockey / \n",
"14-Guitar / \n",
"12-Food / 32-Recipe / 226-Dessert / 239-Roasting / 295-Chocolate / 770-Banana / 1085-Barbecue grill / \n",
"3-Concert / 68-Lighting / \n",
"2671-Digital electronics / \n",
"0-Games / 1-Vehicle / 2-Video game / 19-PC game / 33-Weapon / 51-Strategy video game / 271-Tank / 409-World of Tanks / \n",
"0-Games / 54-Highlight film / 60-Basketball / 81-Athlete / 168-Basketball moves / \n",
"6-Animation / 15-Cartoon / 89-Comics / 123-Naruto / 272-iPod / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"0-Games / 5-Dance / 191-Cheerleading / \n",
"6-Animation / 15-Cartoon / 123-Naruto / 285-Sasuke Uchiha / \n",
"3-Concert / 98-Festival / \n",
"9-Music video / \n",
"0-Games / 60-Basketball / 666-Concrete / \n",
"1-Vehicle / 68-Lighting / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 263-Light / 393-Unmanned aerial vehicle / 604-Quadcopter / \n",
"25-Toy / 290-Pony / 343-My Little Pony / 417-Play-Doh / \n",
"3-Concert / \n",
"10-Animal / 550-Dinosaur / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 252-Tractor pulling / 585-Lawn / 741-Mower / 805-Lawn mower / \n",
"1100-Hat / 3817-Garage sale / \n",
"641-Mask / 2242-Pillow / \n",
"9-Music video / 163-Running / \n",
"0-Games / 49-School / 60-Basketball / 168-Basketball moves / \n",
"10-Animal / 18-Outdoor recreation / 42-Fishing / 91-Fish / 147-River / 510-Fishing lure / 844-Bass / \n",
"10-Animal / 88-Machine / 319-Robot / 354-Egg / \n",
"5-Dance / \n",
"3-Concert / 5-Dance / \n",
"5-Dance / \n",
"47-Personal computer / 1045-Computer case / 2481-Cooler Master / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / \n",
"99-Christmas / 2424-Greeting card / \n",
"1-Vehicle / 4-Car / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"3-Concert / 68-Lighting / \n",
"8-Football / \n",
"0-Games / 2-Video game / 55-Video game console / 386-Xbox / \n",
"3-Concert / \n",
"0-Games / 90-Sports game / 180-FIFA 15 / \n",
"205-Newscaster / 273-News program / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 70-Driving / 2261-Nissan Silvia / \n",
"9-Music video / \n",
"28-Fashion / 206-Shoe / 333-Nike; Inc. / 368-Sneakers / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / \n",
"1-Vehicle / 308-Ship / \n",
"0-Games / 2-Video game / 47-Personal computer / 192-Battlefield / 602-Central processing unit / 1996-Intel Core i7 / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 111-PlayStation 3 / 413-PlayStation 2 / 3263-Shadow of the Colossus / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 398-The Legend of Zelda / \n",
"9-Music video / \n",
"0-Games / 60-Basketball / 168-Basketball moves / 255-Slam dunk / 1101-Point guard / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 94-Dashcam / 1415-Kia Motors / 4060-Kia Sportage / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 217-Hunting / 1215-Wild boar / \n",
"83-Skateboarding / 108-Skateboard / \n",
"237-Final Fantasy / 730-Final Fantasy VII / 1857-Sephiroth / 2041-Crisis Core: Final Fantasy VII / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 297-Grand Theft Auto IV / \n",
"6-Animation / 15-Cartoon / 61-Art / 97-Drawing / \n",
"5-Dance / 31-Disc jockey / 78-Wedding / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 787-Monster truck / \n",
"0-Games / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 299-Volkswagen Passenger Cars / 526-Volkswagen / 711-Gran Turismo / 953-Gran Turismo 5 / 1519-Gran Turismo 6 / 1761-Volkswagen Jetta / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 704-Call of Duty: World at War / \n",
"0-Games / 6-Animation / 15-Cartoon / 877-Hatsune Miku: Project DIVA / \n",
"298-Television advertisement / 337-Advertising / 640-Bottle / 720-Wine / \n",
"1033-Drill / 1226-Metalworking / 1995-Hammer / \n",
"861-Mixing console / \n",
"164-Beach / 447-Sand / \n",
"5-Dance / 16-Performance art / 95-Talent show / \n",
"2389-Exercise ball / \n",
"15-Cartoon / 61-Art / 97-Drawing / 408-Sketch / \n",
"4490-Nier / \n",
"1-Vehicle / 4-Car / \n",
"28-Fashion / 56-Hair / 919-Hair conditioner / \n",
"3-Concert / 5-Dance / 16-Performance art / 412-Monster / \n",
"15-Cartoon / 89-Comics / 123-Naruto / \n",
"0-Games / 2-Video game / 1000-AdventureQuest Worlds / \n",
"2-Video game / 420-The Elder Scrolls V: Skyrim / 441-The Elder Scrolls / \n",
"0-Games / 530-Cue sports / 568-Pool / 994-Cue stick / 2524-Carom billiards / \n",
"1-Vehicle / 4-Car / 34-Action-adventure game / 1661-Volkswagen Gol / \n",
"0-Games / 90-Sports game / 180-FIFA 15 / \n",
"14-Guitar / 345-Loudspeaker / 689-Harmonica / 738-Guitar amplifier / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"3-Concert / 14-Guitar / \n",
"7-Musician / 36-Piano / 53-Keyboard / 878-Composer / \n",
"66-Bollywood / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 118-Grand Theft Auto V / 663-Street / \n",
"9-Music video / \n",
"2227-Multimedios Televisión / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 270-Sedan / \n",
"25-Toy / 323-Toddler / \n",
"46-Choir / \n",
"46-Choir / \n",
"1-Vehicle / 27-Motorcycle / 114-Engine / \n",
"5-Dance / \n",
"14-Guitar / 63-Acoustic guitar / 2744-Bajo sexto / \n",
"311-Street Fighter / 496-Super Street Fighter IV / \n",
"3-Concert / 16-Performance art / \n",
"8-Football / \n",
"1-Vehicle / 18-Outdoor recreation / 87-Boat / 225-Lake / 233-Ocean / \n",
"1-Vehicle / 4-Car / 303-Paint / \n",
"8-Football / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / \n",
"126-The Walt Disney Company / 2904-Express train / \n",
"2102-IKEA / \n",
"9-Music video / \n",
"75-Wrestling / \n",
"14-Guitar / 24-String instrument / 639-Fiddle / 670-Banjo / \n",
"13-Musical ensemble / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"6-Animation / 25-Toy / 477-Transformers / 990-Optimus Prime / \n",
"3-Concert / 16-Performance art / \n",
"6-Animation / 15-Cartoon / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / 864-Speedometer / 2543-2001 GSX-R1000 / \n",
"13-Musical ensemble / \n",
"12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / 109-Cooking show / 179-Kitchen / 557-Salad / \n",
"12-Food / 86-Plant / 137-Tree / 348-Fruit / 770-Banana / \n",
"10-Animal / 80-Horse / 203-Marching band / 217-Hunting / \n",
"415-Sewing / 476-Textile / 1479-Quilt / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 76-Ball / 81-Athlete / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 708-Mermaid / 1092-Game Boy Advance / \n",
"0-Games / 35-Minecraft / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 1072-Total War / \n",
"31-Disc jockey / \n",
"5-Dance / 889-Swing / \n",
"56-Hair / 583-Wig / \n",
"6-Animation / 15-Cartoon / 61-Art / 97-Drawing / 123-Naruto / 183-Manga / 408-Sketch / \n",
"75-Wrestling / 269-Comic book / 313-Book / \n",
"8-Football / 623-Coach / \n",
"6-Animation / 15-Cartoon / \n",
"90-Sports game / 99-Christmas / 180-FIFA 15 / 466-Gift / \n",
"1-Vehicle / 4-Car / 680-Subaru / 1286-Subaru / \n",
"47-Personal computer / 97-Drawing / 145-Tablet computer / 189-iPad / \n",
"31-Disc jockey / 3458-DMC World DJ Championships / \n",
"1-Vehicle / 4-Car / 449-Honda / \n",
"12-Food / 66-Bollywood / \n",
"0-Games / 60-Basketball / 93-Comedy / \n",
"59-Winter sport / 84-Snow / 177-Skiing / 281-Snowboarding / 305-Ski / 501-Snowboard / \n",
"1-Vehicle / 4-Car / 40-Road / 64-Transport / 74-Truck / 114-Engine / 259-Fire / 371-Firefighter / 575-Traffic / 663-Street / 796-Fire engine / 1020-Emergency vehicle / \n",
"1-Vehicle / 25-Toy / 62-Train / 119-Locomotive / 507-Thomas the Tank Engine / 1279-Plarail / \n",
"12-Food / 539-Home appliance / 818-Washing machine / 1142-Laundry / 1456-Clothes dryer / \n",
"5-Dance / 9-Music video / \n",
"6-Animation / 15-Cartoon / 126-The Walt Disney Company / 412-Monster / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 207-Exhaust system / 618-Fiat Automobiles / \n",
"1-Vehicle / 11-Motorsport / 40-Road / 59-Winter sport / 84-Snow / 120-Winter / 610-Snowmobile / 1201-Ski-Doo / \n",
"99-Christmas / 137-Tree / 964-Christmas tree / 2776-Pine / \n",
"1-Vehicle / 50-Aircraft / 88-Machine / 1257-Turbine / 2194-Wind power / 2441-Wind chime / \n",
"3-Concert / \n",
"7-Musician / 9-Music video / 14-Guitar / \n",
"1649-Fast food / \n",
"9-Music video / \n",
"12-Food / \n",
"115-Ballet / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 81-Athlete / \n",
"98-Festival / \n",
"330-Money / \n",
"1-Vehicle / 4-Car / 10-Animal / 48-Pet / 71-Dog / \n",
"13-Musical ensemble / 296-Saxophone / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / \n",
"1026-Foreign Exchange Market / \n",
"1-Vehicle / 4-Car / 1461-Electric car / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 76-Ball / 81-Athlete / \n",
"6-Animation / 15-Cartoon / 1674-Ouran High School Host Club / \n",
"5-Dance / 16-Performance art / 678-Latin dance / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 124-Tractor / 252-Tractor pulling / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / 1825-Cajón / \n",
"12-Food / 122-Weight training / \n",
"743-Edward Cullen / \n",
"5-Dance / \n",
"1-Vehicle / 50-Aircraft / 223-Weather / \n",
"6-Animation / 350-Circus / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"1-Vehicle / 4-Car / 406-Chevrolet / 1502-Chrysler / \n",
"162-Medicine / 2372-Dentist / \n",
"903-Mass Effect / 4061-Mass Effect / \n",
"8-Football / \n",
"110-Album / \n",
"6-Animation / 15-Cartoon / 123-Naruto / 183-Manga / 4598-Beavis / 4622-Butt-head / \n",
"6-Animation / \n",
"6-Animation / 15-Cartoon / \n",
"6-Animation / 20-Trailer / \n",
"6-Animation / 25-Toy / 140-LEGO / 264-Star Wars / 852-Lego Star Wars / 940-Clay animation / \n",
"6-Animation / 15-Cartoon / 61-Art / 290-Pony / 343-My Little Pony / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 51-Strategy video game / 1072-Total War / 2674-Rome: Total War / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 135-Drifting / \n",
"0-Games / 2-Video game / 686-Diablo III / 872-Diablo / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / \n",
"31-Disc jockey / \n",
"9-Music video / \n",
"31-Disc jockey / \n",
"1-Vehicle / 4-Car / 1510-Liquid / 2303-Motor oil / \n",
"223-Weather / \n",
"1-Vehicle / 50-Aircraft / 149-Model aircraft / \n",
"0-Games / 2-Video game / 2572-Lupin III / \n",
"676-Handbag / 829-Bead / \n",
"6-Animation / 15-Cartoon / 89-Comics / 294-Batman / 630-Superman / 643-Hero / 748-Iron Man / 843-The Doctor / \n",
"5-Dance / 16-Performance art / \n",
"78-Wedding / 190-Bride / 421-Gown / 590-Wedding dress / \n",
"5-Dance / 59-Winter sport / 112-Ice skating / 262-Ice / 316-Figure skating / 632-Ice dancing / \n",
"1013-Walking / \n",
"267-Mountain / 347-Climbing / 4563-Mont Blanc / \n",
"12-Food / 86-Plant / 348-Fruit / 4021-Pomegranate / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"0-Games / 240-Arcade game / 1802-Jubeat / \n",
"0-Games / 8-Football / 54-Highlight film / 65-Kick / 73-Ball / 76-Ball / \n",
"0-Games / 23-Mobile phone / 35-Minecraft / \n",
"46-Choir / 219-Accordion / 318-Church / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 73-Ball / \n",
"6-Animation / 10-Animal / 15-Cartoon / 723-Pig / \n",
"1-Vehicle / 4-Car / 299-Volkswagen Passenger Cars / 526-Volkswagen / 668-Volkswagen Golf / \n",
"0-Games / 116-Pokémon / 186-Pokémon / 1387-Pokémon Battle Revolution / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 70-Driving / \n",
"159-World of Warcraft / 171-Warcraft / 811-Mouse / \n",
"9-Music video / \n",
"6-Animation / 25-Toy / 200-Doll / 323-Toddler / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 77-Arena / \n",
"3-Concert / 7-Musician / 39-Drums / \n",
"110-Album / \n",
"10-Animal / 12-Food / 132-Bird / 540-Insect / 3017-Beetle / \n",
"0-Games / 142-Hockey / \n",
"8-Football / \n",
"61-Art / 97-Drawing / 148-Painting / 528-Pencil / 628-Portrait / 890-Brush / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 24-String instrument / 38-Orchestra / 150-Violin / 450-Viola / \n",
"20-Trailer / \n",
"0-Games / 1-Vehicle / 2-Video game / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"9-Music video / \n",
"3-Concert / 68-Lighting / 263-Light / \n",
"10-Animal / 25-Toy / 48-Pet / 166-Cat / 346-Kitten / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 138-Call of Duty: Black Ops II / \n",
"38-Orchestra / \n",
"18-Outdoor recreation / 22-Nature / 86-Plant / 137-Tree / \n",
"9-Music video / \n",
"0-Games / 1232-Perfect World / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 151-Landing / 188-Airline / 1547-Passenger / \n",
"429-Website / \n",
"0-Games / 211-RuneScape / 1873-Crossbow / \n",
"0-Games / 2-Video game / 33-Weapon / 35-Minecraft / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 30-Drummer / 38-Orchestra / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 83-Skateboarding / 108-Skateboard / 272-iPod / 331-iPod touch / \n",
"170-Home improvement / 1671-Drywall / \n",
"12-Food / 179-Kitchen / \n",
"8-Football / \n",
"0-Games / 2-Video game / 21-Stadium / 60-Basketball / 79-American football / 90-Sports game / 176-PlayStation 4 / 314-Xbox One / 561-Madden NFL / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 595-Champion / \n",
"98-Festival / 250-Parade / 253-Carnival / \n",
"6-Animation / \n",
"110-Album / \n",
"31-Disc jockey / \n",
"275-Diving / 287-Human swimming / 329-Swimming pool / \n",
"10-Animal / 48-Pet / 132-Bird / \n",
"5-Dance / 16-Performance art / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / \n",
"0-Games / 8-Football / \n",
"28-Fashion / 206-Shoe / 333-Nike; Inc. / 368-Sneakers / 1423-Nike Air Max / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / \n",
"130-Gymnastics / \n",
"3-Concert / 7-Musician / \n",
"5-Dance / 49-School / 95-Talent show / 191-Cheerleading / 306-High school / \n",
"476-Textile / 3822-Mold / \n",
"1-Vehicle / 113-House / 707-Roof / 1168-Neighbourhood / \n",
"0-Games / 2-Video game / 19-PC game / 55-Video game console / \n",
"0-Games / 2-Video game / 55-Video game console / 199-Wii / \n",
"0-Games / 599-Dofus / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 226-Dessert / 389-Restaurant / 684-Ice cream / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"153-Dress / 1535-Prize / \n",
"20-Trailer / 66-Bollywood / 1059-Movie theater / \n",
"6-Animation / 15-Cartoon / \n",
"264-Star Wars / 1393-Lightsaber / \n",
"0-Games / 308-Ship / 605-Cruise ship / \n",
"3-Concert / 5-Dance / 98-Festival / \n",
"5-Dance / 76-Ball / 130-Gymnastics / 1451-Ribbon / \n",
"0-Games / 781-Pachinko / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 3881-Porsche 930 / \n",
"0-Games / 2-Video game / 159-World of Warcraft / 171-Warcraft / 217-Hunting / \n",
"163-Running / \n",
"6-Animation / 123-Naruto / \n",
"10-Animal / 48-Pet / 132-Bird / \n",
"567-Compact disc / \n",
"49-School / 293-Slide show / \n",
"9-Music video / 79-American football / 209-University / \n",
"36-Piano / 53-Keyboard / 3953-Kawai Musical Instruments / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 553-Sony Xperia / 3867-Sony Ericsson Xperia arc / \n",
"79-American football / \n",
"12-Food / 26-Cooking / 32-Recipe / 236-Baking / 442-Bread / 485-Dough / 600-Flour / 4022-Bun / \n",
"1-Vehicle / 4-Car / 234-Ford / 1634-Ford Fiesta / \n",
"0-Games / 2-Video game / 55-Video game console / 292-Handheld game console / 320-Super Smash Bros. / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"61-Art / 97-Drawing / 148-Painting / 303-Paint / 2124-Spray painting / \n",
"75-Wrestling / \n",
"0-Games / 2-Video game / 19-PC game / 229-Halo / 1161-Halo 2 / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / \n",
"0-Games / 2-Video game / 43-Call of Duty / 138-Call of Duty: Black Ops II / \n",
"1-Vehicle / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 431-Call of Duty: Advanced Warfare / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"6-Animation / 25-Toy / \n",
"2-Video game / 55-Video game console / 292-Handheld game console / 317-PlayStation / 362-PlayStation Portable / 596-Monster Hunter / 682-PlayStation Vita / 1339-Monster Hunter Freedom Unite / \n",
"219-Accordion / 655-Diatonic button accordion / \n",
"93-Comedy / \n",
"31-Disc jockey / \n",
"0-Games / 33-Weapon / 257-Counter-Strike / 1109-Counter-Strike Online / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / \n",
"1-Vehicle / 4-Car / 344-Coupé / 2517-Chevrolet Opala / \n",
"5-Dance / \n",
"12-Food / 500-Rice / 2000-Mango / \n",
"55-Video game console / 127-Xbox 360 / 386-Xbox / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 854-Diesel engine / 2824-Diesel locomotive / \n",
"0-Games / 21-Stadium / 54-Highlight film / 562-Pitcher / 946-Baseball park / \n",
"110-Album / \n",
"0-Games / 2-Video game / 730-Final Fantasy VII / 1312-Dynasty Warriors / \n",
"5-Dance / 892-Aerobics / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"99-Christmas / 558-Santa Claus / \n",
"0-Games / 2267-Geometry Dash / 2347-Marble / 2870-Marble / \n",
"6-Animation / 15-Cartoon / 398-The Legend of Zelda / 877-Hatsune Miku: Project DIVA / \n",
"12-Food / 26-Cooking / 32-Recipe / 301-Meal / \n",
"2037-Straw / \n",
"5-Dance / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"31-Disc jockey / \n",
"45-Cosmetics / \n",
"56-Hair / 340-Afro-textured hair / \n",
"5-Dance / 16-Performance art / 978-Love song / \n",
"1909-Analog synthesizer / \n",
"230-Brass instrument / 296-Saxophone / 1006-Tenor saxophone / \n",
"46-Choir / \n",
"0-Games / 51-Strategy video game / 165-League of Legends / \n",
"20-Trailer / 1469-X-Men / 3507-Superhero movie / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 84-Snow / 129-Aviation / 139-Airport / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / 472-Boeing 737 / 1145-Winter storm / 1529-Jet engine / \n",
"9-Music video / 45-Cosmetics / \n",
"0-Games / 33-Weapon / 664-CrossFire / \n",
"3-Concert / 16-Performance art / \n",
"1-Vehicle / 4-Car / 167-Four-wheel drive / 197-Trail / 198-Off-road vehicle / 277-Sport utility vehicle / 327-Mud / \n",
"5-Dance / 9-Music video / \n",
"22-Nature / \n",
"3-Concert / 68-Lighting / 98-Festival / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 555-Longboard / 667-Longboarding / 784-Honey / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"6-Animation / 15-Cartoon / 1160-Giant panda / \n",
"14-Guitar / 1231-Mandolin / \n",
"1-Vehicle / 74-Truck / 249-Heavy equipment / 976-Demolition / \n",
"768-Diamond / \n",
"689-Harmonica / \n",
"1-Vehicle / 22-Nature / 62-Train / 94-Dashcam / 103-Rail transport / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / \n",
"1-Vehicle / 4-Car / 517-Audi / \n",
"0-Games / 47-Personal computer / 141-Microsoft Windows / 4072-Outlast / \n",
"1-Vehicle / 11-Motorsport / 18-Outdoor recreation / 27-Motorcycle / 105-Motorcycling / 197-Trail / 198-Off-road vehicle / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 172-Motocross / \n",
"1874-Mattress / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"315-Knife / \n",
"3-Concert / \n",
"36-Piano / 53-Keyboard / 104-Pianist / \n",
"10-Animal / 18-Outdoor recreation / 42-Fishing / 91-Fish / 601-Shark / 3292-Rock fishing / \n",
"6-Animation / 15-Cartoon / 887-King Koopa / \n",
"10-Animal / 1716-Rat / \n",
"1-Vehicle / 4-Car / 270-Sedan / 299-Volkswagen Passenger Cars / 526-Volkswagen / \n",
"1-Vehicle / 4-Car / 88-Machine / 114-Engine / \n",
"0-Games / 49-School / 60-Basketball / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / 176-PlayStation 4 / 216-Call of Duty: Modern Warfare 3 / \n",
"59-Winter sport / 112-Ice skating / 262-Ice / 316-Figure skating / 833-Ice skate / \n",
"0-Games / 60-Basketball / \n",
"3-Concert / \n",
"10-Animal / \n",
"47-Personal computer / 145-Tablet computer / 189-iPad / 1150-iPad 2 / \n",
"122-Weight training / 214-Muscle / \n",
"6-Animation / 1809-Tokyo Mew Mew / \n",
"8-Football / \n",
"1-Vehicle / 4-Car / 64-Transport / 286-Bus / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 70-Driving / 349-Chevrolet / 406-Chevrolet / 509-Classic car / 647-Muscle car / 2061-Chevrolet Chevelle / \n",
"23-Mobile phone / 101-Telephone / 986-Suit / 2191-Text messaging / \n",
"0-Games / 2-Video game / 33-Weapon / \n",
"59-Winter sport / 84-Snow / 120-Winter / 177-Skiing / 278-GoPro / 281-Snowboarding / 305-Ski / 501-Snowboard / \n",
"5-Dance / 95-Talent show / \n",
"1-Vehicle / 4-Car / 659-Van / \n",
"0-Games / 2-Video game / 35-Minecraft / 308-Ship / 2767-Captain Hook / \n",
"9-Music video / \n",
"20-Trailer / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / \n",
"47-Personal computer / \n",
"1-Vehicle / 50-Aircraft / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 809-Logo / \n",
"9-Music video / \n",
"6-Animation / 15-Cartoon / \n",
"47-Personal computer / 145-Tablet computer / 189-iPad / 1464-iPad Mini / \n",
"9-Music video / 31-Disc jockey / \n",
"122-Weight training / \n",
"20-Trailer / \n",
"978-Love song / \n",
"1-Vehicle / 4-Car / 531-Hatchback / \n",
"10-Animal / 48-Pet / 166-Cat / 2859-Black cat / \n",
"380-Assassin's Creed / 1585-Assassin's Creed Unity / \n",
"10-Animal / 48-Pet / 166-Cat / 346-Kitten / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / 911-Studio / \n",
"28-Fashion / \n",
"3-Concert / \n",
"6-Animation / 4714-Air Gear / \n",
"0-Games / 141-Microsoft Windows / \n",
"3-Concert / \n",
"3-Concert / \n",
"162-Medicine / \n",
"196-Eye / \n",
"0-Games / 1029-Aion: The Tower of Eternity / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 74-Truck / 154-Wheel / 158-Tire / 234-Ford / 375-Pickup truck / 1493-Supercharger / 2427-Ford F-Series / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 67-Cycling / 69-Bicycle / 220-Mountain bike / \n",
"98-Festival / 1453-Balloon / 1879-Hot air balloon / 1972-Toy balloon / \n",
"0-Games / 2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 4424-Nexus One / \n",
"12-Food / 26-Cooking / 32-Recipe / 208-Wood / 274-Meat / 589-Grilling / 620-Barbecue / 1256-Smoking / \n",
"45-Cosmetics / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 3190-Chevrolet Cruze / \n",
"28-Fashion / \n",
"61-Art / 457-Knitting / 546-Stitch / 653-Thread / 1795-Sock / 1804-Sewing needle / \n",
"820-Bagpipes / \n",
"10-Animal / 80-Horse / 247-Stallion / 289-Dressage / 404-Livestock / \n",
"3-Concert / 68-Lighting / 133-Nightclub / \n",
"88-Machine / 208-Wood / 1700-Lumber / \n",
"10-Animal / 571-Bear / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 105-Motorcycling / \n",
"22-Nature / 213-Forest / \n",
"14-Guitar / 458-Gold / \n",
"523-Cattle / 534-Milk / \n",
"0-Games / 2-Video game / 51-Strategy video game / \n",
"0-Games / 2-Video game / 47-Personal computer / 127-Xbox 360 / 4134-Ni no Kuni: Wrath of the White Witch / \n",
"31-Disc jockey / \n",
"3996-Fandango / \n",
"2-Video game / 276-Room / \n",
"3-Concert / \n",
"7-Musician / 13-Musical ensemble / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 42-Fishing / 91-Fish / 147-River / 510-Fishing lure / \n",
"68-Lighting / \n",
"3850-Weighing scale / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"2-Video game / \n",
"0-Games / 2-Video game / \n",
"214-Muscle / \n",
"3-Concert / 68-Lighting / \n",
"10-Animal / 80-Horse / 384-Horse racing / \n",
"1-Vehicle / 4-Car / 40-Road / 94-Dashcam / 1960-Traffic light / \n",
"1-Vehicle / 28-Fashion / 663-Street / 1168-Neighbourhood / \n",
"3-Concert / 5-Dance / \n",
"13-Musical ensemble / 376-Flute / 1326-Recorder / \n",
"1-Vehicle / 250-Parade / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 114-Engine / 135-Drifting / 1887-Tricycle / 3614-Drift trikes / \n",
"296-Saxophone / 1139-Belle / 1265-Kyle Kingson / \n",
"0-Games / 2-Video game / 55-Video game console / 199-Wii / 456-Go-kart / 518-Mario Kart / \n",
"624-Number / 1818-Calculator / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 239-Roasting / 1381-Garlic / \n",
"66-Bollywood / \n",
"5-Dance / \n",
"5-Dance / \n",
"66-Bollywood / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 150-Violin / 264-Star Wars / \n",
"121-Photography / 628-Portrait / \n",
"93-Comedy / 125-Television / \n",
"9-Music video / \n",
"14-Guitar / \n",
"0-Games / 2-Video game / 19-PC game / 1196-Dragon Age / 1735-Dragon Age: Inquisition / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 47-Personal computer / 145-Tablet computer / 178-Samsung Galaxy / 487-Samsung Electronics / 1287-Samsung Galaxy Tab series / \n",
"0-Games / 8-Football / \n",
"913-President / \n",
"0-Games / 2-Video game / 51-Strategy video game / 165-League of Legends / 1494-Jungle / \n",
"1-Vehicle / 1036-Moped / \n",
"1513-Hitman / 2470-Hitman: Absolution / \n",
"1453-Balloon / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 19-PC game / 34-Action-adventure game / 118-Grand Theft Auto V / 127-Xbox 360 / \n",
"0-Games / 6-Animation / 1946-Total Drama Island / \n",
"5-Dance / 16-Performance art / 115-Ballet / \n",
"5-Dance / \n",
"3653-Masih ad-Dajjal / \n",
"303-Paint / 323-Toddler / 324-Finger / \n",
"6-Animation / 15-Cartoon / 538-Illustration / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"7-Musician / 30-Drummer / 39-Drums / 44-Drums / 46-Choir / 128-Cymbal / 146-Snare drum / \n",
"6-Animation / 15-Cartoon / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"0-Games / 2-Video game / 43-Call of Duty / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / 422-Call of Duty: Ghosts / \n",
"5-Dance / 16-Performance art / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 207-Exhaust system / \n",
"25-Toy / 106-Hairstyle / 200-Doll / 439-Barbie / \n",
"46-Choir / \n",
"67-Cycling / 69-Bicycle / 377-BMX bike / \n",
"99-Christmas / 466-Gift / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / \n",
"3-Concert / 2738-Fender Jazz Bass / \n",
"6-Animation / 15-Cartoon / 61-Art / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / \n",
"3-Concert / 7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / \n",
"14-Guitar / 63-Acoustic guitar / 100-Electric guitar / \n",
"1-Vehicle / 50-Aircraft / 778-Gliding / \n",
"219-Accordion / \n",
"102-Building / 113-House / 353-Wall / 433-Construction / 666-Concrete / \n",
"10-Animal / 48-Pet / 71-Dog / 166-Cat / 182-Puppy / \n",
"0-Games / 2-Video game / 14-Guitar / 445-Guitar Hero / \n",
"1572-Husband / \n",
"505-Mega Man / 2602-Mega Man X / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / 1562-Viola caipira / \n",
"8-Football / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 393-Unmanned aerial vehicle / \n",
"9-Music video / 31-Disc jockey / \n",
"9-Music video / \n",
"25-Toy / 354-Egg / 410-Teacher / 417-Play-Doh / 611-Kinder Surprise / \n",
"1043-Router / \n",
"45-Cosmetics / 238-Nail / 300-Nail art / 304-Nail polish / 338-Manicure / \n",
"5-Dance / \n",
"25-Toy / 455-Unidentified flying object / 1152-Plush / 1212-Stuffed toy / 1668-Claw crane / \n",
"10-Animal / 540-Insect / 775-Bee / \n",
"22-Nature / 141-Microsoft Windows / \n",
"110-Album / \n",
"13-Musical ensemble / 38-Orchestra / 203-Marching band / 634-Trombone / 886-James Bond / \n",
"10-Animal / 132-Bird / 718-Duck / 1947-Goose / \n",
"0-Games / 8-Football / \n",
"10-Animal / 12-Food / 26-Cooking / 91-Fish / 893-Fish as food / 2125-Fillet / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / \n",
"0-Games / 6-Animation / 23-Mobile phone / 29-Smartphone / 553-Sony Xperia / 4440-Sony Ericsson Xperia X8 / \n",
"66-Bollywood / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 6-Animation / 432-Kingdom Hearts / 665-Kingdom Hearts / 836-Sora / 941-Kingdom Hearts II / 2779-Universe of Kingdom Hearts / \n",
"5-Dance / 16-Performance art / 59-Winter sport / 95-Talent show / 112-Ice skating / 115-Ballet / 316-Figure skating / \n",
"75-Wrestling / 117-Boxing / \n",
"0-Games / 2043-Mabinogi / \n",
"0-Games / 2-Video game / 19-PC game / 2334-Phantasy Star Online 2 / \n",
"12-Food / 1162-Dietary supplement / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 932-Microsoft Lumia / \n",
"0-Games / 960-Badminton / \n",
"0-Games / 2-Video game / 39-Drums / 445-Guitar Hero / 724-Rock Band / 1762-Rock Band 3 / \n",
"0-Games / 5-Dance / 49-School / 191-Cheerleading / \n",
"83-Skateboarding / 108-Skateboard / 221-Skatepark / 703-Roller skating / 1937-Inline skates / \n",
"22-Nature / 279-Garden / \n",
"0-Games / 739-Guild Wars / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 16-Performance art / 39-Drums / 44-Drums / \n",
"12-Food / 323-Toddler / 500-Rice / 2928-Cereal / \n",
"315-Knife / 676-Handbag / \n",
"0-Games / 489-Board game / 799-Chess / 922-Tabletop game / \n",
"61-Art / 97-Drawing / 148-Painting / 159-World of Warcraft / 171-Warcraft / \n",
"7-Musician / 13-Musical ensemble / 24-String instrument / 150-Violin / 401-Cello / 450-Viola / \n",
"9-Music video / \n",
"6-Animation / 15-Cartoon / 4504-Xenosaga / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"36-Piano / 46-Choir / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"18-Outdoor recreation / 20-Trailer / 22-Nature / 86-Plant / \n",
"9-Music video / \n",
"3-Concert / 7-Musician / 14-Guitar / 24-String instrument / \n",
"122-Weight training / \n",
"66-Bollywood / \n",
"3-Concert / 16-Performance art / 3599-The Wonderful Wizard of Oz / \n",
"38-Orchestra / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 96-Soldier / 134-Call of Duty: Black Ops / \n",
"3-Concert / 7-Musician / 36-Piano / 104-Pianist / \n",
"5-Dance / \n",
"3-Concert / 21-Stadium / 77-Arena / \n",
"148-Painting / 2538-Painter / \n",
"1-Vehicle / 4-Car / 114-Engine / 1491-Steam engine / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / 207-Exhaust system / \n",
"16-Performance art / 93-Comedy / 355-Comedian / \n",
"9-Music video / \n",
"0-Games / 10-Animal / 48-Pet / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"3-Concert / 7-Musician / 24-String instrument / 44-Drums / 1442-Tabla / \n",
"1731-Haunted house / \n",
"10-Animal / 550-Dinosaur / \n",
"113-House / \n",
"319-Robot / 4438-Humanoid robot / \n",
"0-Games / 2-Video game / 2880-Worms / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 1247-Penalty kick / \n",
"0-Games / 2-Video game / 6-Animation / 685-Luigi / 3506-Luigi's Mansion / \n",
"214-Muscle / 661-Stretching / \n",
"264-Star Wars / \n",
"43-Call of Duty / 134-Call of Duty: Black Ops / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 1333-Wangan Midnight / \n",
"762-Angel / \n",
"5-Dance / 16-Performance art / \n",
"6-Animation / 15-Cartoon / 264-Star Wars / 2576-Clone trooper / \n",
"22-Nature / 499-Cloud / \n",
"12-Food / 26-Cooking / 32-Recipe / 236-Baking / 560-Sugar / 578-Cookie / 867-Tea / 1081-Lemon / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 553-Sony Xperia / 1097-Touchscreen / 3619-Xperia Play / \n",
"439-Barbie / \n",
"0-Games / 2-Video game / 83-Skateboarding / 90-Sports game / 1708-Skate 3 / \n",
"227-Water / \n",
"15-Cartoon / \n",
"12-Food / 389-Restaurant / 462-Table / \n",
"10-Animal / 22-Nature / 86-Plant / 91-Fish / 258-Aquarium / \n",
"83-Skateboarding / 221-Skatepark / \n",
"3-Concert / \n",
"0-Games / 35-Minecraft / \n",
"7-Musician / 14-Guitar / 24-String instrument / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 352-Need for Speed / 473-Ford Mustang / 2739-Midnight Club: Los Angeles / \n",
"2641-Radha / \n",
"3009-LED lamp / \n",
"23-Mobile phone / 29-Smartphone / 3623-BlackBerry Z10 / \n",
"1-Vehicle / 4-Car / 207-Exhaust system / \n",
"1219-Ariel / \n",
"10-Animal / 22-Nature / 588-Reptile / 1354-Frog / \n",
"1002-String quartet / \n",
"86-Plant / 136-Gardening / 468-Biology / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 70-Driving / 277-Sport utility vehicle / 332-Mercedes-Benz / \n",
"0-Games / 8-Football / 595-Champion / \n",
"0-Games / 19-PC game / 192-Battlefield / 383-Battlefield 4 / \n",
"10-Animal / 1354-Frog / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 96-Soldier / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / 194-Call of Duty: Modern Warfare 2 / 216-Call of Duty: Modern Warfare 3 / 422-Call of Duty: Ghosts / 431-Call of Duty: Advanced Warfare / \n",
"0-Games / 60-Basketball / \n",
"10-Animal / 71-Dog / 217-Hunting / 1037-Grass / \n",
"147-River / \n",
"0-Games / 2-Video game / 43-Call of Duty / 96-Soldier / 302-Call of Duty 4: Modern Warfare / 635-Achievement / \n",
"3-Concert / 7-Musician / 14-Guitar / 30-Drummer / 39-Drums / \n",
"342-Windows Media Video / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / \n",
"1-Vehicle / 4-Car / 344-Coupé / 683-Renault / 1388-V6 engine / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 210-Cookware and bakeware / \n",
"9-Music video / 310-The Sims / 543-The Sims 2 / \n",
"1-Vehicle / 4-Car / 40-Road / 64-Transport / 74-Truck / 328-Simulation video game / 1099-Euro Truck Simulator 2 / 3886-Mercedes-Benz Actros / \n",
"2-Video game / \n",
"2685-Giant / \n",
"1927-Atari / \n",
"0-Games / 51-Strategy video game / 373-Clash of Clans / \n",
"6-Animation / 264-Star Wars / 717-Darth Vader / \n",
"45-Cosmetics / 238-Nail / 300-Nail art / 304-Nail polish / 307-Hand / 324-Finger / 338-Manicure / 823-Artificial nails / \n",
"97-Drawing / 920-Chair / \n",
"0-Games / 2-Video game / 111-PlayStation 3 / 127-Xbox 360 / 176-PlayStation 4 / 181-Xbox / 314-Xbox One / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 352-Need for Speed / 1770-Need for Speed: Hot Pursuit / 3146-Need for Speed: Undercover / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / 4680-A&E / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"8-Football / 623-Coach / \n",
"7-Musician / 36-Piano / 53-Keyboard / 104-Pianist / \n",
"36-Piano / 376-Flute / 446-Sheet music / 878-Composer / 1326-Recorder / 1942-Oboe / \n",
"12-Food / 358-Rugby football / \n",
"6-Animation / 20-Trailer / 1266-Death Note / \n",
"3-Concert / 13-Musical ensemble / 38-Orchestra / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 84-Snow / 118-Grand Theft Auto V / 198-Off-road vehicle / \n",
"10-Animal / 48-Pet / 132-Bird / \n",
"3362-WildStar / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 88-Machine / 114-Engine / 154-Wheel / 158-Tire / \n",
"3480-Sound system / \n",
"0-Games / 2-Video game / 626-Puzzle / 4392-The Evil Within / \n",
"20-Trailer / 2368-Marvel Universe / 3276-Punisher / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 297-Grand Theft Auto IV / 1611-Garbage truck / \n",
"25-Toy / 140-LEGO / 735-Kamen Rider Series / \n",
"85-Combat / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / \n",
"6-Animation / 22-Nature / 877-Hatsune Miku: Project DIVA / \n",
"28-Fashion / 1525-Wallet / \n",
"233-Ocean / 672-Bodyboarding / \n",
"0-Games / 2-Video game / 6-Animation / 126-The Walt Disney Company / \n",
"1-Vehicle / 1772-Forklift / 2297-Elf / \n",
"6-Animation / 15-Cartoon / \n",
"61-Art / 97-Drawing / 148-Painting / 479-Graffiti / 1362-Canvas / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / \n",
"1-Vehicle / 27-Motorcycle / 40-Road / \n",
"3-Concert / 30-Drummer / 39-Drums / 44-Drums / 2656-Guitar Center / \n",
"18-Outdoor recreation / 96-Soldier / 217-Hunting / \n",
"1-Vehicle / 455-Unidentified flying object / \n",
"9-Music video / 14-Guitar / \n",
"9-Music video / \n",
"8-Football / 54-Highlight film / 79-American football / \n",
"1-Vehicle / 4-Car / 6-Animation / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / 154-Wheel / \n",
"1-Vehicle / 50-Aircraft / 778-Gliding / 1200-Ultralight aviation / \n",
"222-Surfing / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 76-Ball / \n",
"0-Games / 2-Video game / 33-Weapon / 96-Soldier / 229-Halo / 638-Gears of War / 773-Gears of War / \n",
"53-Keyboard / 397-Organ / 646-Pipe organ / \n",
"162-Medicine / \n",
"46-Choir / \n",
"512-Printing / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 475-Cheese / 2500-Pear / 3382-Risotto / \n",
"10-Animal / 132-Bird / 4362-Turmeric / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 105-Motorcycling / 172-Motocross / 197-Trail / \n",
"1-Vehicle / 27-Motorcycle / 207-Exhaust system / 336-Scooter / \n",
"1-Vehicle / \n",
"3-Concert / 5-Dance / 191-Cheerleading / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"0-Games / 8-Football / 21-Stadium / 73-Ball / \n",
"3-Concert / 5-Dance / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"121-Photography / 155-Camera / 566-Digital camera / \n",
"398-The Legend of Zelda / 1228-The Legend of Zelda: Ocarina of Time / \n",
"122-Weight training / 204-Gym / \n",
"25-Toy / 140-LEGO / \n",
"670-Banjo / \n",
"1-Vehicle / 4-Car / 449-Honda / 1614-Honda Accord / \n",
"0-Games / 116-Pokémon / 186-Pokémon / \n",
"10-Animal / 45-Cosmetics / 48-Pet / 166-Cat / \n",
"1-Vehicle / 4-Car / 70-Driving / 94-Dashcam / 552-Honda Civic / \n",
"3-Concert / \n",
"1-Vehicle / 1573-United States Army / \n",
"6-Animation / \n",
"162-Medicine / 342-Windows Media Video / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / 38-Orchestra / 150-Violin / 639-Fiddle / \n",
"85-Combat / 1832-Hand-to-hand combat / \n",
"61-Art / 97-Drawing / 512-Printing / 1063-Ink / \n",
"0-Games / 8-Football / 54-Highlight film / 65-Kick / 73-Ball / 90-Sports game / \n",
"36-Piano / 53-Keyboard / 104-Pianist / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / 915-Town / \n",
"355-Comedian / \n",
"12-Food / 86-Plant / 136-Gardening / 227-Water / 279-Garden / 363-Soil / 1720-Grape / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 179-Kitchen / 184-Vegetable / 454-Sauce / 752-Pasta / 771-Noodle / 1539-Ramen / 2178-Chinese food / \n",
"0-Games / 59-Winter sport / 112-Ice skating / 142-Hockey / 312-Ice rink / \n",
"12-Food / 32-Recipe / 367-Drink / 720-Wine / 786-Juice / 904-Cocktail / 1065-Bartender / 1669-Vodka / 2406-Peach / \n",
"0-Games / 2-Video game / 33-Weapon / 664-CrossFire / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"9-Music video / \n",
"1-Vehicle / 62-Train / 68-Lighting / 152-Track / 161-Train station / 428-Rail transport modelling / 1450-Model building / \n",
"6-Animation / 9-Music video / 15-Cartoon / \n",
"0-Games / 90-Sports game / 180-FIFA 15 / \n",
"38-Orchestra / 376-Flute / \n",
"45-Cosmetics / 56-Hair / \n",
"0-Games / 2-Video game / 43-Call of Duty / 111-PlayStation 3 / 194-Call of Duty: Modern Warfare 2 / \n",
"4343-Canon EOS 700D / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 508-iPhone 5 / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 2-Video game / 47-Personal computer / 2869-Bomberman / \n",
"8-Football / \n",
"122-Weight training / 204-Gym / 214-Muscle / 532-Barbell / 1250-Biceps / 1264-Head / 1389-Dumbbell / 2045-Biceps curl / \n",
"85-Combat / 117-Boxing / \n",
"5-Dance / 115-Ballet / 1253-Ballet Dancer / \n",
"0-Games / 1-Vehicle / 2-Video game / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 96-Soldier / 194-Call of Duty: Modern Warfare 2 / 302-Call of Duty 4: Modern Warfare / \n",
"0-Games / 2-Video game / 886-James Bond / 4485-Warhawk / \n",
"45-Cosmetics / 238-Nail / 300-Nail art / 304-Nail polish / 338-Manicure / \n",
"0-Games / 792-DarkOrbit / \n",
"10-Animal / 71-Dog / 2049-Police dog / \n",
"1697-Astrological sign / \n",
"85-Combat / 1327-Minnie Mouse / \n",
"18-Outdoor recreation / 59-Winter sport / 84-Snow / 120-Winter / 177-Skiing / 281-Snowboarding / 305-Ski / \n",
"47-Personal computer / 131-Computer / \n",
"12-Food / \n",
"15-Cartoon / \n",
"0-Games / 2-Video game / 515-Dodge / 2527-Loadout / \n",
"12-Food / 1510-Liquid / \n",
"1-Vehicle / 4-Car / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"75-Wrestling / \n",
"1-Vehicle / 4-Car / 864-Speedometer / 1286-Subaru / 3040-Subaru Legacy / \n",
"185-Dragon Ball / \n",
"1-Vehicle / 33-Weapon / 64-Transport / 271-Tank / \n",
"10-Animal / 48-Pet / 71-Dog / 458-Gold / 3173-Great Dane / \n",
"35-Minecraft / \n",
"0-Games / 3227-I Wanna Be the Guy / \n",
"1-Vehicle / 4-Car / 15-Cartoon / \n",
"4583-TV9 / \n",
"9-Music video / \n",
"78-Wedding / \n",
"13-Musical ensemble / 38-Orchestra / \n",
"8-Football / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 728-Dirt track racing / \n",
"12-Food / 26-Cooking / 32-Recipe / 226-Dessert / 232-Cake / 236-Baking / 437-Cream / 759-Icing / \n",
"47-Personal computer / 228-Laptop / 3000-ThinkPad / \n",
"0-Games / 2-Video game / 264-Star Wars / 714-Jedi / 2117-Star Wars Battlefront / 2340-Star Wars: Battlefront II / \n",
"9-Music video / \n",
"0-Games / 35-Minecraft / \n",
"807-Seed / \n",
"25-Toy / 2525-Gashapon / 3396-Ressha Sentai ToQger / \n",
"6-Animation / 10-Animal / 15-Cartoon / 61-Art / 89-Comics / 290-Pony / 343-My Little Pony / \n",
"6-Animation / 7-Musician / 9-Music video / \n",
"1992-Space Shuttle / \n",
"14-Guitar / 100-Electric guitar / \n",
"25-Toy / 89-Comics / 435-Action figure / 2121-Marvel Legends / 3276-Punisher / \n",
"3-Concert / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 73-Ball / \n",
"0-Games / 8-Football / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 81-Athlete / \n",
"122-Weight training / 390-Model / \n",
"10-Animal / 970-Monkey / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 101-Telephone / 3441-HTC HD2 / \n",
"0-Games / 2-Video game / 102-Building / 113-House / 174-Farm / 310-The Sims / 328-Simulation video game / 339-Architecture / 564-The Sims 3 / \n",
"8-Football / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 57-Race track / 94-Dashcam / 135-Drifting / 2288-Ford Escort / \n",
"46-Choir / \n",
"9-Music video / \n",
"38-Orchestra / \n",
"37-Gadget / 141-Microsoft Windows / \n",
"0-Games / 2-Video game / 6-Animation / 34-Action-adventure game / \n",
"415-Sewing / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 927-Supermoto / \n",
"0-Games / 413-PlayStation 2 / 849-Tales / \n",
"36-Piano / \n",
"1-Vehicle / 4-Car / 154-Wheel / 201-Grand Theft Auto: San Andreas / 299-Volkswagen Passenger Cars / 493-Rim / \n",
"227-Water / \n",
"3-Concert / 7-Musician / \n",
"6-Animation / 10-Animal / 15-Cartoon / 1354-Frog / \n",
"477-Transformers / 990-Optimus Prime / 1900-Autobot / \n",
"110-Album / 322-Mixtape / \n",
"0-Games / 2-Video game / 8-Football / 21-Stadium / 65-Kick / 76-Ball / 90-Sports game / 180-FIFA 15 / \n",
"9-Music video / \n",
"33-Weapon / 261-Tool / 315-Knife / 673-Saw / \n",
"1-Vehicle / 4-Car / 114-Engine / 245-BMW / 827-Pump / 1233-BMW 3 Series / 1622-BMW M5 / 2305-BMW 5 Series / 2736-BMW 5 Series (E39) / 3097-Thermostat / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 70-Driving / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 169-Rallying / 1321-Škoda Auto / \n",
"256-Tennis / 974-Racket / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / 265-Electronic keyboard / 592-Synthesia / \n",
"28-Fashion / 206-Shoe / 333-Nike; Inc. / 368-Sneakers / \n",
"28-Fashion / 45-Cosmetics / 335-Nintendo 3DS / \n",
"3-Concert / 357-DVD / \n",
"5-Dance / 13-Musical ensemble / 78-Wedding / \n",
"61-Art / 200-Doll / \n",
"315-Knife / \n",
"189-iPad / \n",
"3-Concert / \n",
"6-Animation / 15-Cartoon / 126-The Walt Disney Company / 616-The Bible / \n",
"5-Dance / \n",
"137-Tree / 208-Wood / 279-Garden / 2887-Swing / \n",
"259-Fire / 371-Firefighter / \n",
"22-Nature / 137-Tree / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 329-Swimming pool / 1081-Lemon / \n",
"6-Animation / \n",
"885-Bell / \n",
"5-Dance / 6-Animation / 440-Lion / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"262-Ice / 981-Sculpture / 1073-Chainsaw / 2339-Swan / \n",
"31-Disc jockey / \n",
"0-Games / 2-Video game / 734-DayZ / \n",
"22-Nature / 1606-Butterfly / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 247-Stallion / 289-Dressage / 290-Pony / 351-Jumping / 404-Livestock / \n",
"9-Music video / \n",
"18-Outdoor recreation / 67-Cycling / 69-Bicycle / 377-BMX bike / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"9-Music video / 78-Wedding / 121-Photography / \n",
"9-Music video / \n",
"1-Vehicle / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 70-Driving / 344-Coupé / 945-MINI Cooper / 1221-MINI / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / 738-Guitar amplifier / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / 44-Drums / \n",
"1-Vehicle / 11-Motorsport / 447-Sand / 975-Dune / \n",
"56-Hair / 106-Hairstyle / 405-Braid / 909-Updo / 1064-French braid / 1292-Ponytail / \n",
"2882-Gladiator / \n",
"1-Vehicle / 4-Car / 245-BMW / \n",
"973-Nintendo 64 / \n",
"1-Vehicle / 4-Car / 449-Honda / 552-Honda Civic / \n",
"2-Video game / \n",
"1-Vehicle / 10-Animal / 18-Outdoor recreation / 42-Fishing / 87-Boat / 91-Fish / 147-River / 225-Lake / 268-Recreational fishing / 844-Bass / 896-Largemouth bass / 2836-Walleye / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"0-Games / 2-Video game / 111-PlayStation 3 / 127-Xbox 360 / 386-Xbox / \n",
"23-Mobile phone / 47-Personal computer / 72-iPhone / 131-Computer / 819-World Wide Web / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"0-Games / 2-Video game / 43-Call of Duty / 194-Call of Duty: Modern Warfare 2 / \n",
"99-Christmas / 137-Tree / 365-Fireworks / 964-Christmas tree / \n",
"3-Concert / 1431-Madison Square Garden / \n",
"176-PlayStation 4 / 386-Xbox / \n",
"79-American football / \n",
"9-Music video / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 47-Personal computer / 241-Fighting game / 486-M.U.G.E.N / 4413-Ultimate Mortal Kombat 3 / \n",
"0-Games / 159-World of Warcraft / 171-Warcraft / 1213-World of Warcraft: Wrath of the Lich King / \n",
"208-Wood / \n",
"0-Games / 2267-Geometry Dash / \n",
"5-Dance / \n",
"0-Games / 703-Roller skating / 3013-Roller derby / \n",
"223-Weather / 267-Mountain / 1058-Volcano / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 237-Final Fantasy / 2506-Final Fantasy XIII-2 / \n",
"10-Animal / 80-Horse / \n",
"9-Music video / \n",
"61-Art / 479-Graffiti / 1385-Street art / \n",
"59-Winter sport / 177-Skiing / \n",
"1181-Mars / \n",
"56-Hair / 871-Hair coloring / \n",
"10-Animal / 48-Pet / 71-Dog / 824-Terrier / 2388-Bull Terrier / 3052-Staffordshire Bull Terrier / \n",
"5-Dance / 16-Performance art / \n",
"7-Musician / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / \n",
"1-Vehicle / 4-Car / 154-Wheel / 650-Convertible / 2016-Volvo Cars / \n",
"31-Disc jockey / \n",
"10-Animal / 22-Nature / 91-Fish / 275-Diving / 288-Underwater / 381-Underwater diving / 407-Scuba diving / 716-Coral / \n",
"0-Games / 20-Trailer / 191-Cheerleading / 199-Wii / \n",
"9-Music video / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 274-Meat / 893-Fish as food / 1115-Steak / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 96-Soldier / 192-Battlefield / 736-ARMA / \n",
"36-Piano / 339-Architecture / \n",
"9-Music video / 1514-Crab / \n",
"3-Concert / 6-Animation / \n",
"455-Unidentified flying object / 502-Sky / \n",
"1-Vehicle / \n",
"1-Vehicle / 519-Recreational vehicle / 1376-Safari / \n",
"1-Vehicle / 4-Car / 64-Transport / 286-Bus / \n",
"5-Dance / \n",
"12-Food / 812-Plastic / 1362-Canvas / \n",
"5-Dance / \n",
"8-Football / 79-American football / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 161-Train station / 224-Rapid transit / \n",
"31-Disc jockey / \n",
"23-Mobile phone / 25-Toy / 29-Smartphone / 88-Machine / 154-Wheel / 319-Robot / \n",
"15-Cartoon / 379-Touhou Project / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 77-Arena / \n",
"23-Mobile phone / 29-Smartphone / 387-Spider-Man / 3561-The Amazing Spider-Man / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 271-Tank / 409-World of Tanks / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / \n",
"5-Dance / 78-Wedding / 190-Bride / 421-Gown / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 96-Soldier / 257-Counter-Strike / \n",
"1-Vehicle / 4-Car / 158-Tire / \n",
"574-Elevator / 1746-Otis Elevator Company / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / 100-Electric guitar / \n",
"102-Building / 174-Farm / 534-Milk / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 382-Dota 2 / 549-Defense of the Ancients / \n",
"25-Toy / 434-Ninja / 951-Teenage Mutant Ninja Turtles / \n",
"3-Concert / \n",
"1-Vehicle / 87-Boat / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"2-Video game / \n",
"1-Vehicle / 50-Aircraft / 102-Building / 2326-Royal Air Force / \n",
"512-Printing / 525-Printer / 1063-Ink / 2634-Inkjet printing / \n",
"516-Headphones / \n",
"0-Games / 17-Racing / 19-PC game / 837-Roblox / \n",
"819-World Wide Web / \n",
"0-Games / 2-Video game / 19-PC game / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"227-Water / 1119-Sensor / 2472-Tap / \n",
"61-Art / 242-Paper / 370-Origami / 2604-Envelope / \n",
"23-Mobile phone / 29-Smartphone / 178-Samsung Galaxy / \n",
"907-Necklace / 1082-Knot / \n",
"0-Games / 88-Machine / 535-Slot machine / \n",
"126-The Walt Disney Company / 144-Amusement park / 576-Walt Disney World / \n",
"3-Concert / 7-Musician / 36-Piano / 53-Keyboard / 104-Pianist / \n",
"66-Bollywood / \n",
"5-Dance / 115-Ballet / \n",
"1-Vehicle / 124-Tractor / 156-Agriculture / 363-Soil / 906-Sowing / \n",
"0-Games / 2-Video game / 67-Cycling / 69-Bicycle / \n",
"28-Fashion / 45-Cosmetics / 195-Mascara / 196-Eye / \n",
"703-Roller skating / \n",
"9-Music video / 31-Disc jockey / 293-Slide show / \n",
"827-Pump / \n",
"0-Games / 2-Video game / 1690-Warhammer Fantasy Battle / \n",
"0-Games / 21-Stadium / 60-Basketball / 77-Arena / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 159-World of Warcraft / 171-Warcraft / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 152-Track / 154-Wheel / 428-Rail transport modelling / \n",
"5-Dance / 16-Performance art / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 85-Combat / 380-Assassin's Creed / 1048-Assassin's Creed / 1140-Assassin's Creed III / 1435-Ezio Auditore da Firenze / \n",
"0-Games / 2-Video game / 19-PC game / 257-Counter-Strike / \n",
"0-Games / 2-Video game / \n",
"0-Games / 8-Football / 90-Sports game / 176-PlayStation 4 / 180-FIFA 15 / \n",
"6-Animation / 379-Touhou Project / 3528-Imperishable Night / \n",
"1528-Euro / \n",
"9-Music video / \n",
"10-Animal / 3710-Kangaroo / 4253-Koala / \n",
"1-Vehicle / 4-Car / 40-Road / 64-Transport / 74-Truck / 250-Parade / 796-Fire engine / 827-Pump / 1020-Emergency vehicle / \n",
"5-Dance / 16-Performance art / \n",
"3-Concert / 5-Dance / \n",
"0-Games / 2-Video game / 20-Trailer / 903-Mass Effect / \n",
"7-Musician / 14-Guitar / 24-String instrument / \n",
"0-Games / 102-Building / 1938-Rust / 3418-The Elder Scrolls III: Morrowind / \n",
"1-Vehicle / 22-Nature / 137-Tree / 433-Construction / 794-Bridge / \n",
"31-Disc jockey / 133-Nightclub / 988-Ibiza / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 824-Terrier / \n",
"3-Concert / 16-Performance art / \n",
"5-Dance / 49-School / 95-Talent show / 191-Cheerleading / \n",
"1-Vehicle / 92-Radio-controlled model / 202-Radio-controlled car / \n",
"93-Comedy / 341-Sketch comedy / 364-Sitcom / 2764-Mad / \n",
"165-League of Legends / 1100-Hat / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 105-Motorcycling / 172-Motocross / \n",
"10-Animal / 132-Bird / 3378-Pigeon racing / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 41-Sports car / 297-Grand Theft Auto IV / 658-Grand Theft Auto: The Lost and Damned / \n",
"1841-Path of Exile / \n",
"0-Games / 2-Video game / 19-PC game / 241-Fighting game / 486-M.U.G.E.N / 764-The King of Fighters / \n",
"110-Album / \n",
"6-Animation / 15-Cartoon / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 179-Kitchen / 184-Vegetable / 210-Cookware and bakeware / 301-Meal / 557-Salad / 752-Pasta / 1024-Culinary art / \n",
"6-Animation / 15-Cartoon / 183-Manga / 3239-Hakuōki / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"36-Piano / \n",
"648-Clown / \n",
"1-Vehicle / 10-Animal / 18-Outdoor recreation / 42-Fishing / 87-Boat / 91-Fish / 268-Recreational fishing / 481-Kayak / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 711-Gran Turismo / 2718-Gran Turismo 4 / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 413-PlayStation 2 / 1125-God of War / 1432-Kratos / 1743-God of War / 2754-God of War II / \n",
"3-Concert / 7-Musician / \n",
"1-Vehicle / 87-Boat / 753-Sailboat / \n",
"7-Musician / 13-Musical ensemble / 203-Marching band / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 178-Samsung Galaxy / \n",
"10-Animal / 2867-Wasp / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 96-Soldier / 194-Call of Duty: Modern Warfare 2 / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"6-Animation / \n",
"413-PlayStation 2 / \n",
"34-Action-adventure game / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 209-University / 365-Fireworks / \n",
"8-Football / \n",
"0-Games / 35-Minecraft / 102-Building / \n",
"0-Games / 8-Football / 54-Highlight film / 76-Ball / \n",
"159-World of Warcraft / 171-Warcraft / 1213-World of Warcraft: Wrath of the Lich King / \n",
"8-Football / \n",
"273-News program / \n",
"102-Building / 208-Wood / 279-Garden / 462-Table / \n",
"8-Football / \n",
"3-Concert / \n",
"86-Plant / 136-Gardening / 279-Garden / 906-Sowing / 1022-Chili pepper / \n",
"5-Dance / \n",
"28-Fashion / 45-Cosmetics / 49-School / 206-Shoe / 880-Trousers / \n",
"0-Games / 6-Animation / 3863-Temple Run / \n",
"47-Personal computer / 145-Tablet computer / 900-MacBook / 1086-MacBook Pro / \n",
"12-Food / \n",
"415-Sewing / 830-Bed / \n",
"6-Animation / 15-Cartoon / 20-Trailer / 1164-Cartoon Network / \n",
"1095-Shiva / 1524-Zee Bangla / \n",
"1-Vehicle / 27-Motorcycle / 245-BMW / \n",
"93-Comedy / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 207-Exhaust system / 2272-Fifth Generation Nissan Z-car (Z33) / \n",
"205-Newscaster / \n",
"13-Musical ensemble / \n",
"7-Musician / 36-Piano / 53-Keyboard / 104-Pianist / \n",
"12-Food / 52-Dish / 215-Eating / 3385-Burrito / \n",
"1-Vehicle / \n",
"22-Nature / 225-Lake / \n",
"1-Vehicle / 50-Aircraft / 139-Airport / 151-Landing / 254-Runway / \n",
"0-Games / 756-Tibia / \n",
"0-Games / 2-Video game / 19-PC game / 686-Diablo III / 872-Diablo / \n",
"6-Animation / 1456-Clothes dryer / \n",
"0-Games / 2-Video game / 33-Weapon / 47-Personal computer / 80-Horse / 96-Soldier / \n",
"3-Concert / \n",
"222-Surfing / 225-Lake / 1667-Standup paddleboarding / \n",
"99-Christmas / \n",
"293-Slide show / \n",
"3-Concert / 5-Dance / \n",
"0-Games / 2-Video game / 90-Sports game / 241-Fighting game / 2301-Dead or Alive / \n",
"320-Super Smash Bros. / \n",
"341-Sketch comedy / \n",
"0-Games / 240-Arcade game / 687-Taiko no Tatsujin / \n",
"121-Photography / 155-Camera / 629-Camera lens / \n",
"0-Games / 2-Video game / 21-Stadium / 60-Basketball / 77-Arena / 81-Athlete / 90-Sports game / 168-Basketball moves / 255-Slam dunk / \n",
"3-Concert / 7-Musician / 39-Drums / \n",
"0-Games / 35-Minecraft / \n",
"9-Music video / \n",
"110-Album / \n",
"9-Music video / 310-The Sims / 543-The Sims 2 / 564-The Sims 3 / \n",
"0-Games / 15-Cartoon / 687-Taiko no Tatsujin / \n",
"1-Vehicle / 4-Car / 25-Toy / 92-Radio-controlled model / 167-Four-wheel drive / 202-Radio-controlled car / 2742-Mini 4WD / \n",
"3286-Stunt Performer / \n",
"1-Vehicle / 27-Motorcycle / 522-Helmet / \n",
"664-CrossFire / \n",
"1-Vehicle / 4-Car / 25-Toy / 2742-Mini 4WD / \n",
"86-Plant / 136-Gardening / 279-Garden / \n",
"95-Talent show / \n",
"6-Animation / 3363-Typography / \n",
"209-University / \n",
"78-Wedding / 1460-Herb / \n",
"205-Newscaster / 2642-CNBC / \n",
"9-Music video / 18-Outdoor recreation / \n",
"122-Weight training / 204-Gym / 529-Squat / 532-Barbell / \n",
"12-Food / 329-Swimming pool / 2258-Patio / \n",
"9-Music video / \n",
"33-Weapon / 261-Tool / 315-Knife / 2387-Military tactics / \n",
"573-Paragliding / \n",
"222-Surfing / 762-Angel / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 231-Supercar / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"199-Wii / 737-Joker / 1199-Kinect / \n",
"28-Fashion / 45-Cosmetics / 56-Hair / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 235-Lipstick / 282-Skin / 291-Rouge / 436-Concealer / 3655-Makeup brush / \n",
"28-Fashion / 45-Cosmetics / 282-Skin / \n",
"0-Games / 2-Video game / 229-Halo / 470-Halo 3 / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / \n",
"1-Vehicle / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 154-Wheel / 158-Tire / 493-Rim / \n",
"25-Toy / 229-Halo / 435-Action figure / 749-Figurine / 1161-Halo 2 / 2121-Marvel Legends / \n",
"264-Star Wars / 468-Biology / 1130-Armour / 2576-Clone trooper / \n",
"1-Vehicle / 4-Car / 40-Road / 212-Highway / 519-Recreational vehicle / \n",
"61-Art / 628-Portrait / 873-Acrylic paint / 1362-Canvas / \n",
"0-Games / 2-Video game / 51-Strategy video game / 537-Zoo / 4255-Zoo Tycoon 2 / 4535-Zoo Tycoon / \n",
"1-Vehicle / 4-Car / 40-Road / 64-Transport / 70-Driving / 74-Truck / 94-Dashcam / 212-Highway / 575-Traffic / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 3168-Resistance / \n",
"0-Games / 2-Video game / 359-Harry Potter (Literary Series) / 4502-Harry Potter and the Order of the Phoenix / \n",
"28-Fashion / 206-Shoe / 681-Leather / 757-Boot / 919-Hair conditioner / \n",
"0-Games / 2-Video game / 1087-Donkey Kong / 2036-Donkey Kong Country / 2274-Donkey Kong / 2970-Donkey Kong / \n",
"3-Concert / 7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"159-World of Warcraft / 171-Warcraft / \n",
"45-Cosmetics / 218-Eyelash / 3106-Eyelash extensions / \n",
"23-Mobile phone / 101-Telephone / \n",
"5-Dance / 20-Trailer / \n",
"0-Games / 2-Video game / 240-Arcade game / 311-Street Fighter / 461-Street Fighter IV / 496-Super Street Fighter IV / \n",
"18-Outdoor recreation / 42-Fishing / 222-Surfing / \n",
"1-Vehicle / 62-Train / 64-Transport / 94-Dashcam / \n",
"3-Concert / 16-Performance art / 30-Drummer / \n",
"0-Games / 2-Video game / 237-Final Fantasy / 432-Kingdom Hearts / 665-Kingdom Hearts / 1857-Sephiroth / 2127-Final Fantasy / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 105-Motorcycling / \n",
"0-Games / 942-Silkroad Online / \n",
"0-Games / 2-Video game / 55-Video game console / 199-Wii / 320-Super Smash Bros. / 335-Nintendo 3DS / 443-Wii U / 697-Super Smash Bros. for Nintendo 3DS and Wii U / 782-Super Smash Bros. Melee / \n",
"6-Animation / 15-Cartoon / 571-Bear / \n",
"6-Animation / 20-Trailer / 126-The Walt Disney Company / \n",
"93-Comedy / 355-Comedian / \n",
"93-Comedy / \n",
"376-Flute / 4605-Bansuri / \n",
"165-League of Legends / \n",
"1-Vehicle / 27-Motorcycle / 207-Exhaust system / \n",
"18-Outdoor recreation / 22-Nature / 275-Diving / 288-Underwater / 381-Underwater diving / 407-Scuba diving / 468-Biology / \n",
"9-Music video / 78-Wedding / \n",
"10-Animal / 71-Dog / \n",
"609-Tattoo / \n",
"0-Games / 35-Minecraft / 488-Door / \n",
"3-Concert / 7-Musician / \n",
"67-Cycling / 69-Bicycle / 83-Skateboarding / 108-Skateboard / 555-Longboard / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 169-Rallying / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 38-Orchestra / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 105-Motorcycling / 172-Motocross / 491-Enduro / 669-Extreme sport / \n",
"10-Animal / 12-Food / 48-Pet / 52-Dish / 184-Vegetable / 215-Eating / 636-Rabbit / \n",
"1017-Yo-yo / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 921-Grand Theft Auto: Vice City / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"4559-Federal Senate / \n",
"0-Games / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 216-Call of Duty: Modern Warfare 3 / \n",
"0-Games / 2-Video game / 10-Animal / \n",
"162-Medicine / 784-Honey / \n",
"56-Hair / 798-Shampoo / \n",
"25-Toy / 88-Machine / 140-LEGO / 319-Robot / 357-DVD / 3753-Plotter / \n",
"93-Comedy / \n",
"1-Vehicle / 164-Beach / 233-Ocean / 423-Coast / 573-Paragliding / 1561-Paramotor / \n",
"93-Comedy / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 664-CrossFire / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 55-Video game console / 201-Grand Theft Auto: San Andreas / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / \n",
"0-Games / 10-Animal / 35-Minecraft / 80-Horse / \n",
"8-Football / 1193-Orange / \n",
"3-Concert / 110-Album / \n",
"205-Newscaster / 463-Cricket / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / \n",
"347-Climbing / 828-Rock climbing / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 196-Eye / 218-Eyelash / 235-Lipstick / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 135-Drifting / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 464-First-person Shooter / \n",
"150-Violin / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 774-Galaxy / 1397-Super Mario Galaxy / 1759-Super Mario Galaxy 2 / \n",
"61-Art / 148-Painting / 303-Paint / 873-Acrylic paint / 936-Watercolor paint / 1362-Canvas / 3177-Tulip / \n",
"261-Tool / \n",
"1-Vehicle / 25-Toy / 62-Train / 507-Thomas the Tank Engine / 1279-Plarail / 1685-Toy train / \n",
"66-Bollywood / \n",
"9-Music video / \n",
"133-Nightclub / 559-Night / \n",
"3-Concert / 9-Music video / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 70-Driving / 231-Supercar / 344-Coupé / 517-Audi / 1537-Audi R8 / 1587-Audi Quattro / 2949-V10 engine / \n",
"5-Dance / 16-Performance art / 361-Flamenco / \n",
"25-Toy / 89-Comics / 435-Action figure / 803-DC Comics / \n",
"5-Dance / 16-Performance art / \n",
"3857-Muskmelon / \n",
"12-Food / 747-Retail / 1328-Grocery store / 1782-Supermarket / \n",
"1183-Karaoke box / \n",
"748-Iron Man / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 40-Road / 274-Meat / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 6-Animation / 17-Racing / \n",
"0-Games / 2-Video game / 51-Strategy video game / 72-iPhone / 1120-Tower defense / \n",
"2478-Red hair / \n",
"55-Video game console / 176-PlayStation 4 / 317-PlayStation / 677-Game controller / 1262-NBA 2K14 / 1482-PlayStation Network / 2074-Killzone / \n",
"3-Concert / 46-Choir / 209-University / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 218-Eyelash / 291-Rouge / 436-Concealer / \n",
"83-Skateboarding / 108-Skateboard / \n",
"170-Home improvement / 276-Room / 353-Wall / 2743-Wallpaper / \n",
"0-Games / 51-Strategy video game / 165-League of Legends / \n",
"28-Fashion / 61-Art / \n",
"0-Games / 60-Basketball / 168-Basketball moves / \n",
"14-Guitar / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"3-Concert / 7-Musician / \n",
"14-Guitar / 24-String instrument / 490-Rain / \n",
"1-Vehicle / 4-Car / 40-Road / 64-Transport / 70-Driving / 74-Truck / 94-Dashcam / \n",
"88-Machine / 261-Tool / 972-Lathe / \n",
"10-Animal / 1373-Torero / \n",
"0-Games / 2-Video game / 34-Action-adventure game / \n",
"10-Animal / 22-Nature / 280-Wildlife / \n",
"0-Games / 2-Video game / 9-Music video / 43-Call of Duty / 138-Call of Duty: Black Ops II / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / \n",
"0-Games / 2-Video game / 928-Borderlands 2 / 2613-Borderlands: The Pre-Sequel! / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / \n",
"49-School / 113-House / 131-Computer / 209-University / \n",
"9-Music video / \n",
"22-Nature / 86-Plant / 689-Harmonica / \n",
"8-Football / 65-Kick / \n",
"36-Piano / 53-Keyboard / \n",
"0-Games / 9-Music video / \n",
"13-Musical ensemble / \n",
"28-Fashion / \n",
"23-Mobile phone / 29-Smartphone / 121-Photography / 155-Camera / 566-Digital camera / \n",
"3151-Kermit the Frog / \n",
"0-Games / 2-Video game / 1789-Slender: The Eight Pages / \n",
"7-Musician / 93-Comedy / 355-Comedian / \n",
"25-Toy / 116-Pokémon / 1152-Plush / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 4395-Sim racing / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"66-Bollywood / \n",
"0-Games / 8-Football / 54-Highlight film / 81-Athlete / \n",
"0-Games / 1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 2010-Chart / \n",
"0-Games / 35-Minecraft / 637-Five Nights at Freddy's / \n",
"6-Animation / 123-Naruto / \n",
"3-Concert / 46-Choir / \n",
"89-Comics / \n",
"7-Musician / 13-Musical ensemble / 230-Brass instrument / \n",
"2-Video game / 18-Outdoor recreation / 51-Strategy video game / 144-Amusement park / 321-Roller coaster / 334-Amusement ride / 1293-RollerCoaster Tycoon 3 / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / 49-School / \n",
"25-Toy / 33-Weapon / 140-LEGO / \n",
"3-Concert / 361-Flamenco / \n",
"6-Animation / 25-Toy / 200-Doll / 439-Barbie / 1272-Cabinetry / \n",
"6-Animation / 9-Music video / 15-Cartoon / \n",
"1-Vehicle / 4-Car / 403-Toyota / 1840-Toyota Hilux / \n",
"49-School / \n",
"6-Animation / 477-Transformers / \n",
"242-Paper / 370-Origami / \n",
"5-Dance / 98-Festival / 130-Gymnastics / 250-Parade / 1835-Majorette / \n",
"7-Musician / 1621-Ork / \n",
"0-Games / 159-World of Warcraft / 171-Warcraft / 643-Hero / \n",
"246-Furniture / 276-Room / 416-Apartment / \n",
"1030-Trampoline / 1998-Canon EOS 600D / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 17-Racing / 41-Sports car / 231-Supercar / 352-Need for Speed / 406-Chevrolet / 952-Chevrolet Corvette / 1600-Need for Speed: World / 4135-Chevrolet Corvette C5 Z06 / \n",
"5-Dance / 9-Music video / \n",
"5-Dance / 16-Performance art / \n",
"10-Animal / 17-Racing / 80-Horse / 384-Horse racing / \n",
"1-Vehicle / 4-Car / 154-Wheel / 158-Tire / \n",
"1-Vehicle / 4-Car / 74-Truck / 167-Four-wheel drive / 327-Mud / 375-Pickup truck / \n",
"3-Concert / 5-Dance / 16-Performance art / 98-Festival / 209-University / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 105-Motorcycling / 172-Motocross / \n",
"1225-Antenna / \n",
"5-Dance / 115-Ballet / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / \n",
"0-Games / 2-Video game / 4465-ourWorld / \n",
"78-Wedding / 121-Photography / 633-Photograph / 2269-Wedding photography / \n",
"3-Concert / 68-Lighting / 77-Arena / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 40-Road / 92-Radio-controlled model / 202-Radio-controlled car / 3839-Volkswagen Golf Mk5 / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 57-Race track / 378-Drag racing / 406-Chevrolet / \n",
"1-Vehicle / 87-Boat / \n",
"3-Concert / \n",
"971-Alphabet / \n",
"42-Fishing / 1082-Knot / 1369-Necktie / \n",
"3-Concert / 31-Disc jockey / 68-Lighting / 133-Nightclub / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"15-Cartoon / 877-Hatsune Miku: Project DIVA / 1648-Hatsune Miku: Project DIVA F / 2682-Hatsune Miku: Project DIVA F 2nd / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"1-Vehicle / 11-Motorsport / 124-Tractor / \n",
"1-Vehicle / 144-Amusement park / \n",
"465-Microphone / \n",
"12-Food / 442-Bread / 2928-Cereal / \n",
"5-Dance / 16-Performance art / 95-Talent show / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 176-PlayStation 4 / 181-Xbox / 314-Xbox One / 422-Call of Duty: Ghosts / \n",
"45-Cosmetics / \n",
"419-Earth / \n",
"12-Food / 26-Cooking / 32-Recipe / 274-Meat / \n",
"0-Games / 88-Machine / 240-Arcade game / 387-Spider-Man / 535-Slot machine / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 508-iPhone 5 / \n",
"0-Games / 35-Minecraft / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"3-Concert / \n",
"3-Concert / 31-Disc jockey / 68-Lighting / 133-Nightclub / 227-Water / \n",
"56-Hair / 106-Hairstyle / \n",
"3-Concert / 7-Musician / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"10-Animal / 48-Pet / 132-Bird / \n",
"113-House / \n",
"452-Watch / 627-Clock / \n",
"84-Snow / 120-Winter / 365-Fireworks / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / 224-Rapid transit / \n",
"93-Comedy / \n",
"1-Vehicle / 11-Motorsport / 50-Aircraft / 82-Airplane / 102-Building / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / \n",
"126-The Walt Disney Company / \n",
"8-Football / \n",
"0-Games / 1-Vehicle / 2-Video game / 11-Motorsport / 456-Go-kart / 518-Mario Kart / 2909-Mario Kart 7 / \n",
"34-Action-adventure game / 47-Personal computer / 131-Computer / 201-Grand Theft Auto: San Andreas / \n",
"273-News program / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / 2744-Bajo sexto / \n",
"0-Games / 2-Video game / \n",
"1-Vehicle / 4-Car / 92-Radio-controlled model / 202-Radio-controlled car / \n",
"1088-Infomercial / \n",
"5-Dance / 78-Wedding / 190-Bride / 430-Ballroom dance / \n",
"12-Food / 26-Cooking / 519-Recreational vehicle / \n",
"3-Concert / 16-Performance art / \n",
"36-Piano / \n",
"287-Human swimming / 293-Slide show / 710-Handball / 2273-Polo / 2360-Water polo / \n",
"6-Animation / 113-House / 170-Home improvement / 179-Kitchen / 742-Countertop / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 114-Engine / 650-Convertible / 1521-Jaguar Cars / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"10-Animal / 18-Outdoor recreation / 42-Fishing / 233-Ocean / 1471-Tuna / \n",
"12-Food / 88-Machine / 262-Ice / \n",
"12-Food / 534-Milk / 1617-Fungus / \n",
"45-Cosmetics / 193-Eye liner / 195-Mascara / 218-Eyelash / 235-Lipstick / 291-Rouge / 641-Mask / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 231-Supercar / 352-Need for Speed / 1637-GameCube / 1770-Need for Speed: Hot Pursuit / \n",
"9-Music video / \n",
"1183-Karaoke box / \n",
"0-Games / 2-Video game / 19-PC game / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / \n",
"0-Games / 734-DayZ / 736-ARMA / 1034-ARMA 2 / 1216-DayZ / \n",
"2-Video game / \n",
"0-Games / 6-Animation / 411-MapleStory / \n",
"242-Paper / 370-Origami / 1449-Rubber band / \n",
"1868-Preacher / \n",
"322-Mixtape / \n",
"10-Animal / 80-Horse / \n",
"96-Soldier / \n",
"1-Vehicle / 64-Transport / 74-Truck / 1070-Semi-trailer truck / \n",
"1043-Router / \n",
"0-Games / 366-Dragon / \n",
"31-Disc jockey / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 921-Grand Theft Auto: Vice City / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 1031-Fisheye lens / \n",
"99-Christmas / 729-Christmas decoration / 964-Christmas tree / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 47-Personal computer / 72-iPhone / 145-Tablet computer / 189-iPad / \n",
"0-Games / 2-Video game / 19-PC game / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / 297-Grand Theft Auto IV / \n",
"12-Food / 26-Cooking / \n",
"0-Games / 60-Basketball / 191-Cheerleading / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 2391-American Airlines / \n",
"130-Gymnastics / 350-Circus / 788-Juggling / \n",
"0-Games / 2-Video game / 2658-Mirror's Edge / \n",
"66-Bollywood / \n",
"102-Building / 113-House / 170-Home improvement / 279-Garden / \n",
"3-Concert / 133-Nightclub / 925-Brain / \n",
"103-Rail transport / 428-Rail transport modelling / \n",
"5-Dance / 85-Combat / 430-Ballroom dance / 4235-American Kenpo / \n",
"1-Vehicle / 25-Toy / 264-Star Wars / \n",
"0-Games / 2-Video game / 426-Resident Evil / 506-Garry's Mod / 1276-Resident Evil / \n",
"3-Concert / 16-Performance art / \n",
"0-Games / 59-Winter sport / 112-Ice skating / 142-Hockey / 3612-NHL 15 / \n",
"1-Vehicle / 4-Car / \n",
"0-Games / 59-Winter sport / 90-Sports game / 112-Ice skating / 142-Hockey / 2546-NHL 14 / \n",
"1-Vehicle / 4-Car / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 161-Train station / 224-Rapid transit / \n",
"376-Flute / \n",
"13-Musical ensemble / 250-Parade / 820-Bagpipes / \n",
"6-Animation / 717-Darth Vader / \n",
"3-Concert / 5-Dance / 16-Performance art / 115-Ballet / \n",
"3622-Def Jam Recordings / \n",
"102-Building / \n",
"20-Trailer / 541-Black-and-white / \n",
"242-Paper / 370-Origami / 2454-Notebook / \n",
"3-Concert / \n",
"113-House / 170-Home improvement / 303-Paint / 353-Wall / 1350-Plaster / 4139-Stucco / \n",
"5-Dance / 16-Performance art / 980-Solo dance / \n",
"117-Boxing / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 4436-Yamaha YZ450F / \n",
"359-Harry Potter (Literary Series) / \n",
"9-Music video / \n",
"3-Concert / 5-Dance / 31-Disc jockey / 133-Nightclub / \n",
"66-Bollywood / \n",
"162-Medicine / \n",
"28-Fashion / 206-Shoe / 333-Nike; Inc. / 368-Sneakers / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 420-The Elder Scrolls V: Skyrim / 441-The Elder Scrolls / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / \n",
"6-Animation / 15-Cartoon / \n",
"59-Winter sport / 112-Ice skating / 142-Hockey / \n",
"102-Building / 318-Church / 397-Organ / 646-Pipe organ / 870-Chapel / \n",
"322-Mixtape / \n",
"1061-Marriage proposal / \n",
"0-Games / 2-Video game / 55-Video game console / 116-Pokémon / 199-Wii / 1298-Pikachu / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / \n",
"5-Dance / 4004-Tug of war / \n",
"1-Vehicle / 573-Paragliding / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 382-Dota 2 / 549-Defense of the Ancients / 1378-Multiplayer online battle arena / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 179-Kitchen / 1024-Culinary art / 1843-Ham / 3356-Cauliflower / 4153-Gratin / \n",
"9-Music video / \n",
"125-Television / \n",
"42-Fishing / \n",
"341-Sketch comedy / \n",
"9-Music video / \n",
"28-Fashion / 45-Cosmetics / 193-Eye liner / 195-Mascara / 218-Eyelash / \n",
"1-Vehicle / 50-Aircraft / \n",
"93-Comedy / \n",
"5-Dance / 78-Wedding / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 154-Wheel / 349-Chevrolet / 406-Chevrolet / 509-Classic car / 647-Muscle car / 1148-Hot rod / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"0-Games / 2-Video game / 241-Fighting game / 320-Super Smash Bros. / 782-Super Smash Bros. Melee / 973-Nintendo 64 / \n",
"5-Dance / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 210-Cookware and bakeware / \n",
"9-Music video / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"1-Vehicle / 4-Car / 2737-GAZ / \n",
"85-Combat / \n",
"3-Concert / 68-Lighting / \n",
"28-Fashion / 153-Dress / \n",
"0-Games / 8-Football / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"6-Animation / 1139-Belle / 3975-Sleeping Beauty / 4264-Princess Aurora / \n",
"462-Table / 544-Table tennis / \n",
"6-Animation / \n",
"31-Disc jockey / \n",
"12-Food / 1596-Cinnamon / 2475-Spoon / \n",
"323-Toddler / 567-Compact disc / 695-Balloon / \n",
"5-Dance / 16-Performance art / \n",
"113-House / 170-Home improvement / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 696-T-shirt / \n",
"99-Christmas / 120-Winter / 150-Violin / \n",
"125-Television / \n",
"0-Games / 141-Microsoft Windows / \n",
"7-Musician / 14-Guitar / 24-String instrument / \n",
"10-Animal / 91-Fish / 1258-Goldfish / \n",
"164-Beach / 222-Surfing / 233-Ocean / \n",
"66-Bollywood / \n",
"0-Games / 2-Video game / 310-The Sims / 328-Simulation video game / 1092-Game Boy Advance / \n",
"1-Vehicle / 50-Aircraft / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 266-Helicopter / 392-Radio-controlled helicopter / \n",
"22-Nature / 155-Camera / 275-Diving / 278-GoPro / 288-Underwater / 407-Scuba diving / 716-Coral / \n",
"85-Combat / 117-Boxing / 399-Kickboxing / \n",
"6-Animation / 183-Manga / \n",
"9-Music video / \n",
"66-Bollywood / \n",
"3714-Chalk / \n",
"0-Games / 2-Video game / 240-Arcade game / \n",
"0-Games / 2-Video game / 4674-Ape Escape / \n",
"5-Dance / 16-Performance art / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 105-Motorcycling / 172-Motocross / 608-Wheelie / 1520-Minibike / \n",
"117-Boxing / \n",
"355-Comedian / \n",
"107-Musical keyboard / 219-Accordion / \n",
"770-Banana / \n",
"0-Games / 2-Video game / 33-Weapon / 736-ARMA / \n",
"25-Toy / 75-Wrestling / 435-Action figure / 1441-Mattel / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 98-Festival / \n",
"5-Dance / 253-Carnival / \n",
"1-Vehicle / 4-Car / 234-Ford / \n",
"5-Dance / 49-School / 191-Cheerleading / 306-High school / \n",
"0-Games / 1644-ArcheAge / \n",
"12-Food / 215-Eating / 323-Toddler / \n",
"15-Cartoon / \n",
"3-Concert / 3208-Lowlands / \n",
"45-Cosmetics / 218-Eyelash / \n",
"8-Football / 79-American football / \n",
"3-Concert / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"0-Games / 271-Tank / 1167-Tanki Online / \n",
"0-Games / 8-Football / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 734-DayZ / 1612-Multi Theft Auto / \n",
"0-Games / 2-Video game / 90-Sports game / 453-Pro Evolution Soccer / 3526-Pro Evolution Soccer 2009 / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 161-Train station / \n",
"10-Animal / 91-Fish / 814-Pond / 1258-Goldfish / 1568-Koi / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 231-Supercar / 901-Mitsubishi / \n",
"0-Games / 75-Wrestling / \n",
"0-Games / 8-Football / \n",
"28-Fashion / 56-Hair / 106-Hairstyle / \n",
"15-Cartoon / 123-Naruto / 285-Sasuke Uchiha / 1180-Itachi Uchiha / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 17-Racing / 19-PC game / 34-Action-adventure game / 41-Sports car / 111-PlayStation 3 / 118-Grand Theft Auto V / \n",
"38-Orchestra / \n",
"12-Food / 2706-Starbucks / \n",
"1-Vehicle / 4-Car / 40-Road / 94-Dashcam / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 207-Exhaust system / 515-Dodge / \n",
"1-Vehicle / 4-Car / 705-Forza Motorsport / 1530-Forza Motorsport 4 / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 73-Ball / 76-Ball / 77-Arena / 81-Athlete / \n",
"0-Games / 19-PC game / 792-DarkOrbit / \n",
"79-American football / \n",
"28-Fashion / 469-Bag / 676-Handbag / 681-Leather / \n",
"0-Games / \n",
"46-Choir / \n",
"9-Music video / \n",
"372-Playing card / 746-Card manipulation / \n",
"10-Animal / 48-Pet / 166-Cat / 346-Kitten / 2154-Gorilla / 2924-Persian cat / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"0-Games / 2-Video game / 237-Final Fantasy / 432-Kingdom Hearts / 665-Kingdom Hearts / 941-Kingdom Hearts II / 1658-Dissidia Final Fantasy / 1908-Kingdom Hearts Birth by Sleep / 2127-Final Fantasy / 3299-Kingdom Hearts: Chain of Memories / \n",
"0-Games / 60-Basketball / 77-Arena / 81-Athlete / 168-Basketball moves / 255-Slam dunk / \n",
"117-Boxing / \n",
"2478-Red hair / \n",
"153-Dress / \n",
"10-Animal / 132-Bird / 242-Paper / 370-Origami / \n",
"9-Music video / 78-Wedding / \n",
"359-Harry Potter (Literary Series) / \n",
"6-Animation / 396-One Piece / \n",
"0-Games / 351-Jumping / 2401-Wolfenstein / \n",
"1-Vehicle / 50-Aircraft / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 266-Helicopter / 392-Radio-controlled helicopter / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / 466-Gift / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 42-Fishing / 91-Fish / 225-Lake / 268-Recreational fishing / 1306-Carp fishing / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 194-Call of Duty: Modern Warfare 2 / 216-Call of Duty: Modern Warfare 3 / \n",
"12-Food / 226-Dessert / 232-Cake / \n",
"51-Strategy video game / 373-Clash of Clans / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / 30-Drummer / 39-Drums / \n",
"60-Basketball / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / \n",
"0-Games / 2-Video game / 55-Video game console / 292-Handheld game console / 584-Super Mario Bros. / \n",
"885-Bell / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / \n",
"725-Human / \n",
"2488-Fitness Center / \n",
"3-Concert / \n",
"121-Photography / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 116-Pokémon / 4461-Pokémon Stadium / \n",
"0-Games / 2-Video game / 685-Luigi / \n",
"170-Home improvement / 208-Wood / 1700-Lumber / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / 172-Motocross / \n",
"1-Vehicle / 4-Car / 84-Snow / 198-Off-road vehicle / 360-Jeep / 3828-Amphibious ATV / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 41-Sports car / 111-PlayStation 3 / 297-Grand Theft Auto IV / 635-Achievement / \n",
"5-Dance / 10-Animal / 80-Horse / 356-Star / \n",
"144-Amusement park / 321-Roller coaster / \n",
"12-Food / 86-Plant / 137-Tree / 348-Fruit / 1319-Apple / \n",
"259-Fire / 995-Attack on Titan / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / \n",
"12-Food / 2032-Pepsi / \n",
"6-Animation / 15-Cartoon / 97-Drawing / \n",
"0-Games / 35-Minecraft / \n",
"14-Guitar / 100-Electric guitar / 1721-Tetris / \n",
"5-Dance / \n",
"0-Games / 256-Tennis / 544-Table tennis / \n",
"6-Animation / 10-Animal / 15-Cartoon / 538-Illustration / 624-Number / 3105-Numerical digit / \n",
"5-Dance / 13-Musical ensemble / 78-Wedding / \n",
"0-Games / 2-Video game / 43-Call of Duty / \n",
"45-Cosmetics / 56-Hair / 798-Shampoo / 919-Hair conditioner / \n",
"121-Photography / 2107-Grey / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 272-iPod / 331-iPod touch / \n",
"67-Cycling / 69-Bicycle / \n",
"10-Animal / 22-Nature / 366-Dragon / 588-Reptile / 763-Snake / 1444-Lizard / \n",
"18-Outdoor recreation / 283-Rock / 347-Climbing / \n",
"3-Concert / 5-Dance / 68-Lighting / \n",
"66-Bollywood / \n",
"426-Resident Evil / 1068-Resident Evil 4 / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 247-Stallion / 289-Dressage / 351-Jumping / 404-Livestock / 438-Mare / \n",
"20-Trailer / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 84-Snow / 118-Grand Theft Auto V / \n",
"5-Dance / 9-Music video / 28-Fashion / 153-Dress / \n",
"6-Animation / 15-Cartoon / 880-Trousers / 4615-Bakura / \n",
"8-Football / 73-Ball / 76-Ball / 333-Nike; Inc. / \n",
"3-Concert / \n",
"3-Concert / \n",
"1416-Cliff / \n",
"9-Music video / \n",
"47-Personal computer / 131-Computer / 228-Laptop / \n",
"3-Concert / 13-Musical ensemble / 30-Drummer / \n",
"51-Strategy video game / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 35-Minecraft / 165-League of Legends / 506-Garry's Mod / \n",
"0-Games / 88-Machine / 240-Arcade game / 535-Slot machine / 4247-Bullion / \n",
"14-Guitar / 100-Electric guitar / \n",
"9-Music video / \n",
"455-Unidentified flying object / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 40-Road / 172-Motocross / 669-Extreme sport / 1815-Pit bike / \n",
"12-Food / 26-Cooking / 32-Recipe / 236-Baking / 485-Dough / 578-Cookie / 600-Flour / 841-Butter / 1081-Lemon / \n",
"20-Trailer / \n",
"5-Dance / \n",
"10-Animal / 18-Outdoor recreation / 42-Fishing / 91-Fish / 510-Fishing lure / 3224-European perch / \n",
"1-Vehicle / 4-Car / 406-Chevrolet / 509-Classic car / 1418-Model car / \n",
"1-Vehicle / 4-Car / 207-Exhaust system / 680-Subaru / 947-Muffler / \n",
"3-Concert / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 1097-Touchscreen / 4128-Nokia N97 / \n",
"12-Food / 26-Cooking / 32-Recipe / 236-Baking / 442-Bread / \n",
"86-Plant / 136-Gardening / \n",
"641-Mask / 723-Pig / \n",
"227-Water / \n",
"1-Vehicle / 27-Motorcycle / 751-Desert / \n",
"56-Hair / 106-Hairstyle / 340-Afro-textured hair / 405-Braid / 857-Afro / 909-Updo / 954-Dreadlocks / 1812-Hair twists / \n",
"9-Music video / \n",
"6-Animation / 9-Music video / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 135-Drifting / 2518-Dirt 3 / \n",
"3-Concert / 7-Musician / 14-Guitar / 24-String instrument / 743-Edward Cullen / \n",
"0-Games / 2-Video game / 19-PC game / 192-Battlefield / 383-Battlefield 4 / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / 1313-Strum / \n",
"20-Trailer / \n",
"12-Food / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 506-Garry's Mod / \n",
"6-Animation / 15-Cartoon / 851-Digimon / \n",
"3-Concert / \n",
"18-Outdoor recreation / 42-Fishing / 84-Snow / 120-Winter / 262-Ice / \n",
"33-Weapon / 217-Hunting / \n",
"898-Pilates / \n",
"1-Vehicle / 4-Car / 9-Music video / \n",
"1-Vehicle / 42-Fishing / 87-Boat / 92-Radio-controlled model / 202-Radio-controlled car / 308-Ship / 1490-Radio-controlled boat / 3163-Tugboat / \n",
"616-The Bible / \n",
"1-Vehicle / 4-Car / 70-Driving / 94-Dashcam / 277-Sport utility vehicle / 983-Lexus / \n",
"31-Disc jockey / 322-Mixtape / \n",
"102-Building / 113-House / 339-Architecture / \n",
"10-Animal / 48-Pet / 71-Dog / 166-Cat / 182-Puppy / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / \n",
"47-Personal computer / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 1031-Fisheye lens / \n",
"5-Dance / \n",
"541-Black-and-white / \n",
"783-Zee TV / \n",
"31-Disc jockey / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / \n",
"20-Trailer / \n",
"3-Concert / 14-Guitar / \n",
"1054-Vacuum cleaner / \n",
"30-Drummer / 39-Drums / 44-Drums / 1825-Cajón / \n",
"85-Combat / 117-Boxing / \n",
"0-Games / 2-Video game / 577-Penguin / \n",
"467-Chipmunk / \n",
"28-Fashion / 67-Cycling / 69-Bicycle / \n",
"0-Games / 2-Video game / 43-Call of Duty / 111-PlayStation 3 / 194-Call of Duty: Modern Warfare 2 / 216-Call of Duty: Modern Warfare 3 / \n",
"222-Surfing / 2415-Big wave surfing / \n",
"2-Video game / 1188-Uncharted / \n",
"0-Games / 311-Street Fighter / 461-Street Fighter IV / 496-Super Street Fighter IV / 1331-Ken Masters / 3936-Gouken / \n",
"3-Concert / 13-Musical ensemble / \n",
"0-Games / 2-Video game / 51-Strategy video game / \n",
"425-City / \n",
"31-Disc jockey / 68-Lighting / 263-Light / 988-Ibiza / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 24-String instrument / 38-Orchestra / 150-Violin / 450-Viola / 2348-Chrono Trigger / \n",
"28-Fashion / 56-Hair / 106-Hairstyle / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 194-Call of Duty: Modern Warfare 2 / \n",
"31-Disc jockey / 754-Gramophone record / 861-Mixing console / 3458-DMC World DJ Championships / \n",
"12-Food / \n",
"6-Animation / 15-Cartoon / 4183-Tamagotchi / \n",
"31-Disc jockey / \n",
"5-Dance / 115-Ballet / 430-Ballroom dance / 678-Latin dance / \n",
"22-Nature / 36-Piano / 376-Flute / \n",
"12-Food / 770-Banana / \n",
"1-Vehicle / 4-Car / 531-Hatchback / 3197-Citroën C4 / \n",
"8-Football / \n",
"22-Nature / 84-Snow / 99-Christmas / 120-Winter / 729-Christmas decoration / \n",
"0-Games / 25-Toy / 140-LEGO / 319-Robot / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"18-Outdoor recreation / 42-Fishing / 1236-Fishing reel / \n",
"3-Concert / 7-Musician / \n",
"1500-Cherry blossom / 2640-Petal / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 1207-Samsung Galaxy S5 / \n",
"25-Toy / 140-LEGO / \n",
"0-Games / \n",
"1314-Garage / 2196-Boiler / \n",
"1-Vehicle / 286-Bus / 2226-Bus driver / 2850-Chauffeur / \n",
"1465-FOX / \n",
"0-Games / 25-Toy / 323-Toddler / 3303-Table football / \n",
"6-Animation / 183-Manga / 3588-Berserk / \n",
"0-Games / 8-Football / \n",
"0-Games / 51-Strategy video game / \n",
"0-Games / 2-Video game / 10-Animal / 23-Mobile phone / 178-Samsung Galaxy / 899-Samsung Galaxy S III / \n",
"764-The King of Fighters / 961-Sega Genesis / \n",
"6-Animation / 183-Manga / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 837-Roblox / \n",
"0-Games / 2-Video game / 111-PlayStation 3 / 192-Battlefield / 374-Battlefield 3 / \n",
"1-Vehicle / 62-Train / 103-Rail transport / 119-Locomotive / 152-Track / \n",
"6-Animation / 15-Cartoon / 61-Art / 97-Drawing / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"739-Guild Wars / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 38-Orchestra / 46-Choir / \n",
"303-Paint / 696-T-shirt / 793-Shirt / \n",
"75-Wrestling / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 235-Lipstick / 291-Rouge / \n",
"3-Concert / \n",
"402-Jewellery / 859-Beadwork / 907-Necklace / 1300-Earrings / 3428-Costume jewelry / \n",
"10-Animal / 80-Horse / 4694-Cruelty to animals / \n",
"1-Vehicle / 50-Aircraft / 129-Aviation / \n",
"0-Games / 6-Animation / 15-Cartoon / 290-Pony / 343-My Little Pony / \n",
"97-Drawing / 479-Graffiti / \n",
"110-Album / \n",
"3-Concert / 16-Performance art / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / 2073-Sabian / \n",
"1-Vehicle / 4-Car / 509-Classic car / 3350-Rolls-Royce Limited / 4501-The Adventures of Tintin / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 161-Train station / 224-Rapid transit / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"1157-Rope / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"9-Music video / 10-Animal / 22-Nature / 280-Wildlife / 1603-Fox / \n",
"3027-Olive oil / \n",
"9-Music video / \n",
"9-Music video / \n",
"6-Animation / \n",
"20-Trailer / \n",
"359-Harry Potter (Literary Series) / 1436-Draco Malfoy / \n",
"614-Hiking / 1203-Backpacking / \n",
"0-Games / 60-Basketball / 77-Arena / 168-Basketball moves / 255-Slam dunk / \n",
"6-Animation / 10-Animal / 25-Toy / 140-LEGO / 550-Dinosaur / 940-Clay animation / 1126-Tyrannosaurus / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / 70-Driving / 349-Chevrolet / 406-Chevrolet / \n",
"293-Slide show / \n",
"14-Guitar / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 318-Church / 397-Organ / 646-Pipe organ / \n",
"1-Vehicle / 4-Car / 449-Honda / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 59-Winter sport / 1829-Sledding / 4709-Luge / \n",
"0-Games / 199-Wii / 256-Tennis / \n",
"66-Bollywood / \n",
"0-Games / 463-Cricket / \n",
"121-Photography / \n",
"25-Toy / 126-The Walt Disney Company / 749-Figurine / \n",
"7-Musician / 13-Musical ensemble / 24-String instrument / 38-Orchestra / 1747-Erhu / \n",
"0-Games / 2-Video game / 35-Minecraft / 174-Farm / 1108-Mushroom / \n",
"36-Piano / \n",
"916-Spa / 1271-Beauty salon / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 30-Drummer / 39-Drums / 44-Drums / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 81-Athlete / \n",
"3-Concert / 21-Stadium / 366-Dragon / \n",
"0-Games / 2-Video game / 6-Animation / 20-Trailer / 43-Call of Duty / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 178-Samsung Galaxy / 3184-Moto X / \n",
"6-Animation / 25-Toy / 35-Minecraft / 140-LEGO / 1302-The Lego Group / 3899-Lego Minecraft / \n",
"1-Vehicle / 4-Car / 40-Road / 64-Transport / 74-Truck / 332-Mercedes-Benz / 3886-Mercedes-Benz Actros / \n",
"6-Animation / 4708-Visual novel / \n",
"56-Hair / 106-Hairstyle / 583-Wig / 607-Lace / \n",
"8-Football / \n",
"28-Fashion / 45-Cosmetics / 238-Nail / 300-Nail art / 304-Nail polish / 307-Hand / 324-Finger / 338-Manicure / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 70-Driving / \n",
"1721-Tetris / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 2616-Chevrolet S-10 / \n",
"372-Playing card / 746-Card manipulation / \n",
"28-Fashion / 956-Glasses / \n",
"0-Games / 8-Football / 90-Sports game / 180-FIFA 15 / 480-FIFA 13 / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 57-Race track / 169-Rallying / \n",
"9-Music video / 322-Mixtape / \n",
"613-Bar / \n",
"59-Winter sport / 112-Ice skating / 142-Hockey / 162-Medicine / 179-Kitchen / \n",
"1143-Marimba / 1734-Xylophone / \n",
"0-Games / 34-Action-adventure game / 330-Money / \n",
"5-Dance / 31-Disc jockey / 133-Nightclub / \n",
"191-Cheerleading / \n",
"975-Dune / \n",
"9-Music video / \n",
"9-Music video / \n",
"234-Ford / 4005-Mercury / \n",
"6-Animation / 15-Cartoon / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / 1475-Knuckles the Echidna / \n",
"12-Food / 261-Tool / 673-Saw / 1073-Chainsaw / 1346-Wood carving / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 74-Truck / 375-Pickup truck / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 234-Ford / 473-Ford Mustang / 1877-Shelby Mustang / 3755-AC Cobra / \n",
"0-Games / 2-Video game / 19-PC game / 211-RuneScape / 330-Money / \n",
"6-Animation / 9-Music video / \n",
"1-Vehicle / 50-Aircraft / 92-Radio-controlled model / 129-Aviation / 149-Model aircraft / 160-Radio-controlled aircraft / 266-Helicopter / 393-Unmanned aerial vehicle / \n",
"110-Album / \n",
"1-Vehicle / 10-Animal / 18-Outdoor recreation / 42-Fishing / 87-Boat / 91-Fish / 233-Ocean / \n",
"7-Musician / 14-Guitar / \n",
"227-Water / 259-Fire / \n",
"1-Vehicle / 4-Car / 41-Sports car / 332-Mercedes-Benz / \n",
"0-Games / 2-Video game / 35-Minecraft / 55-Video game console / 111-PlayStation 3 / 394-Map / \n",
"5-Dance / 16-Performance art / 95-Talent show / \n",
"85-Combat / 117-Boxing / 204-Gym / 399-Kickboxing / 1239-K-1 / \n",
"6-Animation / 25-Toy / 701-Beyblade / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 1845-Pug / \n",
"85-Combat / 117-Boxing / \n",
"0-Games / 2-Video game / 229-Halo / 470-Halo 3 / 644-Halo: Reach / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"12-Food / 170-Home improvement / 179-Kitchen / 276-Room / 742-Countertop / \n",
"46-Choir / 318-Church / \n",
"25-Toy / 4652-Weekly Shōnen Jump / \n",
"10-Animal / 48-Pet / 71-Dog / \n",
"3-Concert / 7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"0-Games / 2-Video game / 19-PC game / \n",
"28-Fashion / 121-Photography / 155-Camera / 566-Digital camera / 633-Photograph / 721-Digital SLR / \n",
"273-News program / \n",
"1459-Magazine / \n",
"9-Music video / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / \n",
"36-Piano / 53-Keyboard / \n",
"33-Weapon / 261-Tool / 315-Knife / \n",
"137-Tree / 640-Bottle / \n",
"22-Nature / \n",
"0-Games / 1-Vehicle / 4-Car / 17-Racing / 41-Sports car / 70-Driving / 245-BMW / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 447-Sand / 751-Desert / 975-Dune / \n",
"0-Games / 1732-Oni / 4197-Ao Oni / \n",
"0-Games / 19-PC game / 35-Minecraft / \n",
"8-Football / 21-Stadium / \n",
"5-Dance / 95-Talent show / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / 870-Chapel / \n",
"3-Concert / 16-Performance art / \n",
"5-Dance / \n",
"3-Concert / \n",
"31-Disc jockey / \n",
"5-Dance / \n",
"162-Medicine / \n",
"0-Games / 2-Video game / 518-Mario Kart / 3964-Diddy Kong / \n",
"0-Games / 35-Minecraft / \n",
"60-Basketball / 76-Ball / 168-Basketball moves / 1399-Streetball / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / 472-Boeing 737 / \n",
"0-Games / 2-Video game / 847-Angry Birds / 4064-Angry Birds Space / \n",
"0-Games / 2-Video game / 335-Nintendo 3DS / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 559-Night / 1761-Volkswagen Jetta / \n",
"144-Amusement park / 321-Roller coaster / 334-Amusement ride / 366-Dragon / 3794-Legoland / \n",
"1-Vehicle / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 197-Trail / 220-Mountain bike / 284-Mountain biking / \n",
"213-Forest / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 96-Soldier / 111-PlayStation 3 / 2074-Killzone / \n",
"14-Guitar / 100-Electric guitar / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"117-Boxing / \n",
"10-Animal / 91-Fish / 215-Eating / 258-Aquarium / 716-Coral / 1051-Reef aquarium / \n",
"1026-Foreign Exchange Market / \n",
"3-Concert / 7-Musician / 16-Performance art / 350-Circus / 2358-Contortion / \n",
"205-Newscaster / 273-News program / \n",
"3-Concert / \n",
"14-Guitar / 36-Piano / 53-Keyboard / 107-Musical keyboard / 265-Electronic keyboard / \n",
"0-Games / 159-World of Warcraft / 1232-Perfect World / \n",
"1-Vehicle / 68-Lighting / 263-Light / \n",
"164-Beach / 1481-Sunglasses / \n",
"0-Games / 2-Video game / 8-Football / 21-Stadium / 90-Sports game / 180-FIFA 15 / 453-Pro Evolution Soccer / 1719-Pro Evolution Soccer 2015 / \n",
"567-Compact disc / 2447-Guzheng / \n",
"20-Trailer / \n",
"28-Fashion / 45-Cosmetics / 676-Handbag / \n",
"85-Combat / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 359-Harry Potter (Literary Series) / \n",
"6-Animation / 15-Cartoon / 123-Naruto / \n",
"205-Newscaster / 752-Pasta / \n",
"9-Music video / \n",
"93-Comedy / \n",
"5-Dance / 694-Tai chi / \n",
"0-Games / 211-RuneScape / \n",
"5-Dance / 8-Football / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / \n",
"3-Concert / 7-Musician / 14-Guitar / 24-String instrument / \n",
"7-Musician / 13-Musical ensemble / 21-Stadium / 49-School / 54-Highlight film / 203-Marching band / 209-University / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"20-Trailer / 78-Wedding / 196-Eye / \n",
"3-Concert / 68-Lighting / 98-Festival / \n",
"93-Comedy / 125-Television / 341-Sketch comedy / \n",
"0-Games / 6-Animation / 624-Number / 971-Alphabet / \n",
"483-Costume / 3153-Freddy Krueger / \n",
"9-Music video / \n",
"78-Wedding / 232-Cake / \n",
"6-Animation / \n",
"8-Football / \n",
"0-Games / 2-Video game / 43-Call of Duty / 422-Call of Duty: Ghosts / \n",
"66-Bollywood / \n",
"3-Concert / 16-Performance art / \n",
"7-Musician / 9-Music video / 14-Guitar / \n",
"444-Family / \n",
"0-Games / 2-Video game / 6-Animation / 816-Gundam / 1564-Mobile Suit Gundam: Extreme Vs. / \n",
"6-Animation / 15-Cartoon / 61-Art / 97-Drawing / 123-Naruto / \n",
"1-Vehicle / 18-Outdoor recreation / 42-Fishing / 87-Boat / 208-Wood / 481-Kayak / 753-Sailboat / 917-Sail / 1309-Sailing ship / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 1450-Model building / 1497-Wool / 2552-Dyeing / \n",
"5-Dance / 115-Ballet / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 184-Vegetable / \n",
"28-Fashion / 56-Hair / 106-Hairstyle / \n",
"325-Vampire / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 362-PlayStation Portable / 1125-God of War / 1432-Kratos / 1743-God of War / \n",
"0-Games / 1-Vehicle / 2-Video game / 11-Motorsport / 55-Video game console / 199-Wii / 456-Go-kart / 518-Mario Kart / \n",
"12-Food / 354-Egg / 1597-Sausage / \n",
"3-Concert / \n",
"5-Dance / 16-Performance art / \n",
"9-Music video / 68-Lighting / 263-Light / 1016-Glove / 1375-Laser lighting display / \n",
"1-Vehicle / 11-Motorsport / 59-Winter sport / 177-Skiing / 278-GoPro / 305-Ski / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 1377-Golden Retriever / \n",
"3-Concert / 1019-Christ / \n",
"25-Toy / 140-LEGO / \n",
"12-Food / 26-Cooking / 32-Recipe / 109-Cooking show / 179-Kitchen / 2588-Mayonnaise / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 332-Mercedes-Benz / 650-Convertible / 891-Mercedes-AMG / 3995-Mercedes-Benz W124 / \n",
"0-Games / 54-Highlight film / 60-Basketball / 168-Basketball moves / 255-Slam dunk / \n",
"3-Concert / \n",
"489-Board game / 799-Chess / \n",
"5-Dance / 6-Animation / 1527-Pretty Cure / \n",
"337-Advertising / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 70-Driving / 155-Camera / 650-Convertible / 1521-Jaguar Cars / 4386-Jaguar F-Type / \n",
"116-Pokémon / 186-Pokémon / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / 813-Running back / \n",
"1-Vehicle / 74-Truck / 1070-Semi-trailer truck / 2920-Food grain / \n",
"1-Vehicle / 4-Car / 41-Sports car / 231-Supercar / 332-Mercedes-Benz / \n",
"9-Music video / 375-Pickup truck / \n",
"4476-Shipyard / \n",
"88-Machine / 283-Rock / 666-Concrete / 2679-Pavement / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / 38-Orchestra / 1231-Mandolin / \n",
"1-Vehicle / 25-Toy / 62-Train / 1279-Plarail / \n",
"5-Dance / 49-School / \n",
"1-Vehicle / 18-Outdoor recreation / 22-Nature / 87-Boat / 147-River / 614-Hiking / \n",
"1-Vehicle / 4-Car / 114-Engine / \n",
"3-Concert / 30-Drummer / \n",
"122-Weight training / 214-Muscle / \n",
"1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 352-Need for Speed / 2772-Shift 2: Unleashed / \n",
"512-Printing / 525-Printer / 1424-Coupon / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 157-Jet aircraft / 175-Airliner / 388-Fighter aircraft / 758-Air force / 2326-Royal Air Force / \n",
"6-Animation / \n",
"0-Games / 2-Video game / 55-Video game console / 2431-Metal Slug / \n",
"3-Concert / \n",
"88-Machine / 400-Woodturning / \n",
"3-Concert / 68-Lighting / 133-Nightclub / 988-Ibiza / \n",
"0-Games / 19-PC game / 51-Strategy video game / 165-League of Legends / \n",
"9-Music video / \n",
"28-Fashion / 45-Cosmetics / 56-Hair / 78-Wedding / 106-Hairstyle / 405-Braid / 909-Updo / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 247-Stallion / 289-Dressage / 351-Jumping / 438-Mare / \n",
"0-Games / 2-Video game / 1981-Commodore 64 / 3918-ZX Spectrum / \n",
"10-Animal / 91-Fish / 275-Diving / 288-Underwater / 381-Underwater diving / 407-Scuba diving / 468-Biology / \n",
"0-Games / 2-Video game / 20-Trailer / 872-Diablo / 2054-Diablo II / \n",
"1-Vehicle / 4-Car / 84-Snow / 120-Winter / 344-Coupé / \n",
"0-Games / 12-Food / 1223-Pastry / \n",
"14-Guitar / \n",
"25-Toy / 626-Puzzle / 631-Cube / 740-Rubik's Cube / \n",
"5-Dance / 6-Animation / 679-Mickey Mouse / 1327-Minnie Mouse / 2793-Disguise / \n",
"1-Vehicle / 4-Car / 41-Sports car / 231-Supercar / \n",
"25-Toy / 140-LEGO / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 77-Arena / 81-Athlete / \n",
"0-Games / 159-World of Warcraft / 171-Warcraft / 1356-Paladin / \n",
"9-Music video / \n",
"3-Concert / 5-Dance / \n",
"6-Animation / 1887-Tricycle / \n",
"9-Music video / \n",
"0-Games / 211-RuneScape / 1178-Quest / \n",
"0-Games / 2-Video game / 19-PC game / 734-DayZ / 736-ARMA / 1034-ARMA 2 / 1216-DayZ / \n",
"1183-Karaoke box / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 161-Train station / \n",
"61-Art / 232-Cake / 242-Paper / 370-Origami / 795-Cupcake / \n",
"1-Vehicle / 4-Car / 56-Hair / 70-Driving / 106-Hairstyle / \n",
"3-Concert / 99-Christmas / \n",
"6-Animation / 10-Animal / 15-Cartoon / \n",
"3-Concert / 7-Musician / 14-Guitar / 24-String instrument / \n",
"5-Dance / 16-Performance art / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"16-Performance art / 95-Talent show / \n",
"0-Games / 2-Video game / 55-Video game console / \n",
"25-Toy / 113-House / 140-LEGO / 1079-Lego City / 2055-Mansion / \n",
"83-Skateboarding / 108-Skateboard / 555-Longboard / 667-Longboarding / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / 465-Microphone / 1013-Walking / \n",
"6-Animation / 61-Art / 97-Drawing / \n",
"3-Concert / 7-Musician / \n",
"28-Fashion / 402-Jewellery / 907-Necklace / 3428-Costume jewelry / \n",
"45-Cosmetics / 235-Lipstick / \n",
"12-Food / 25-Toy / 389-Restaurant / 417-Play-Doh / 439-Barbie / 485-Dough / 578-Cookie / 944-Hamburger / 962-McDonald's / 1185-Plasticine / 1701-French fries / 3102-Cookie Monster / 3822-Mold / \n",
"12-Food / 367-Drink / 2992-Energy drink / \n",
"0-Games / 2-Video game / 420-The Elder Scrolls V: Skyrim / 441-The Elder Scrolls / \n",
"0-Games / 2-Video game / 43-Call of Duty / 127-Xbox 360 / 134-Call of Duty: Black Ops / 192-Battlefield / 194-Call of Duty: Modern Warfare 2 / 216-Call of Duty: Modern Warfare 3 / 229-Halo / 464-First-person Shooter / 466-Gift / 2081-Infinity Ward / \n",
"130-Gymnastics / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 515-Dodge / 2078-Dodge Challenger / 2530-Chrysler Hemi engine / \n",
"0-Games / 19-PC game / 311-Street Fighter / 496-Super Street Fighter IV / \n",
"7-Musician / 38-Orchestra / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"3-Concert / 5-Dance / 49-School / 95-Talent show / \n",
"14-Guitar / 100-Electric guitar / \n",
"3-Concert / 5-Dance / 46-Choir / \n",
"45-Cosmetics / 238-Nail / 300-Nail art / 304-Nail polish / 307-Hand / 324-Finger / \n",
"5-Dance / 16-Performance art / \n",
"0-Games / 8-Football / 73-Ball / \n",
"0-Games / 2-Video game / 9-Music video / 477-Transformers / 990-Optimus Prime / 2186-Megatron / \n",
"0-Games / 2-Video game / 1637-GameCube / 4347-Rocky Balboa / \n",
"9-Music video / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 172-Motocross / 927-Supermoto / \n",
"3-Concert / \n",
"18-Outdoor recreation / 42-Fishing / 222-Surfing / 1008-Kite / \n",
"142-Hockey / 1610-Field hockey / \n",
"233-Ocean / \n",
"296-Saxophone / 1006-Tenor saxophone / \n",
"66-Bollywood / 1281-Couch / \n",
"3-Concert / 7-Musician / 16-Performance art / 39-Drums / 4412-Alcatraz Island / \n",
"22-Nature / 458-Gold / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 597-Soup / \n",
"23-Mobile phone / 29-Smartphone / 125-Television / \n",
"0-Games / 8-Football / \n",
"0-Games / 5-Dance / 257-Counter-Strike / \n",
"3-Concert / 5-Dance / 7-Musician / 13-Musical ensemble / 46-Choir / 78-Wedding / 99-Christmas / \n",
"10-Animal / 537-Zoo / 970-Monkey / 1948-Ape / \n",
"633-Photograph / \n",
"25-Toy / 200-Doll / 412-Monster / 503-Monster High / 2476-Maze / \n",
"6-Animation / 20-Trailer / \n",
"0-Games / 1-Vehicle / 19-PC game / 33-Weapon / 271-Tank / 409-World of Tanks / \n",
"0-Games / 2-Video game / 25-Toy / 120-Winter / 140-LEGO / 359-Harry Potter (Literary Series) / \n",
"1-Vehicle / 4-Car / 114-Engine / 618-Fiat Automobiles / \n",
"0-Games / 8-Football / \n",
"81-Athlete / 222-Surfing / 815-Surfboard / \n",
"1731-Haunted house / \n",
"130-Gymnastics / 1558-Hula hoop / \n",
"0-Games / 2-Video game / 6-Animation / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 135-Drifting / \n",
"5-Dance / \n",
"227-Water / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 301-Meal / 597-Soup / 2040-Korean food / 3539-Clam / \n",
"36-Piano / 53-Keyboard / \n",
"2-Video game / 320-Super Smash Bros. / 782-Super Smash Bros. Melee / 2289-Super Mario Bros. 3 / \n",
"0-Games / 2-Video game / 6-Animation / 111-PlayStation 3 / 2628-Sly Cooper / \n",
"3-Concert / 7-Musician / 9-Music video / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 1032-BlackBerry / 3732-BlackBerry Bold / \n",
"3-Concert / 7-Musician / \n",
"3-Concert / 68-Lighting / \n",
"12-Food / 26-Cooking / 32-Recipe / 49-School / 52-Dish / 808-Breakfast / 1408-Snack / \n",
"467-Chipmunk / \n",
"0-Games / 60-Basketball / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"1-Vehicle / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 220-Mountain bike / 284-Mountain biking / \n",
"6-Animation / 10-Animal / 15-Cartoon / \n",
"10-Animal / 91-Fish / 258-Aquarium / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 172-Motocross / 278-GoPro / 669-Extreme sport / 3548-Yamaha YZ125 / \n",
"6-Animation / 15-Cartoon / 61-Art / 97-Drawing / \n",
"7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"6-Animation / \n",
"0-Games / 8-Football / \n",
"12-Food / 1208-Coconut / \n",
"1-Vehicle / 4-Car / 18-Outdoor recreation / 25-Toy / 74-Truck / 92-Radio-controlled model / 167-Four-wheel drive / 198-Off-road vehicle / 202-Radio-controlled car / 787-Monster truck / \n",
"10-Animal / 636-Rabbit / \n",
"9-Music video / 323-Toddler / \n",
"10-Animal / 91-Fish / 258-Aquarium / \n",
"6-Animation / 61-Art / 97-Drawing / 148-Painting / 1716-Rat / \n",
"83-Skateboarding / \n",
"8-Football / 76-Ball / \n",
"5-Dance / 16-Performance art / \n",
"14-Guitar / \n",
"28-Fashion / 206-Shoe / \n",
"0-Games / 116-Pokémon / 186-Pokémon / \n",
"7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 774-Galaxy / 1397-Super Mario Galaxy / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 73-Ball / \n",
"3-Concert / 7-Musician / \n",
"3-Concert / 13-Musical ensemble / 30-Drummer / 295-Chocolate / \n",
"122-Weight training / 204-Gym / 214-Muscle / \n",
"22-Nature / 86-Plant / 136-Gardening / 279-Garden / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"0-Games / 256-Tennis / \n",
"6-Animation / \n",
"28-Fashion / 56-Hair / 106-Hairstyle / 405-Braid / 909-Updo / 1292-Ponytail / \n",
"162-Medicine / 468-Biology / \n",
"12-Food / 26-Cooking / 32-Recipe / 442-Bread / 1567-Cook / \n",
"276-Room / 888-Living room / 1739-Fireplace / \n",
"121-Photography / 155-Camera / \n",
"14-Guitar / \n",
"5-Dance / 10-Animal / 80-Horse / 247-Stallion / \n",
"3-Concert / 726-Clarinet / \n",
"46-Choir / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 1588-Prince of Persia / 1653-Prince / 3814-Prince of Persia: Warrior Within / \n",
"93-Comedy / \n",
"56-Hair / 106-Hairstyle / \n",
"0-Games / 2-Video game / 43-Call of Duty / 194-Call of Duty: Modern Warfare 2 / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 73-Ball / 76-Ball / 81-Athlete / \n",
"9-Music video / 233-Ocean / \n",
"8-Football / \n",
"5-Dance / \n",
"1-Vehicle / 88-Machine / 92-Radio-controlled model / 202-Radio-controlled car / 271-Tank / \n",
"3-Concert / 7-Musician / 14-Guitar / 100-Electric guitar / \n",
"20-Trailer / 66-Bollywood / \n",
"1-Vehicle / 4-Car / 40-Road / 94-Dashcam / 212-Highway / 223-Weather / 490-Rain / 958-Thunderstorm / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 44-Drums / 46-Choir / 1019-Christ / \n",
"0-Games / 1-Vehicle / 2-Video game / 286-Bus / \n",
"1-Vehicle / 4-Car / 245-BMW / 1285-Parking / 3475-BMW X6 / \n",
"0-Games / 54-Highlight film / 60-Basketball / 79-American football / \n",
"5-Dance / 6-Animation / 1527-Pretty Cure / \n",
"0-Games / 1-Vehicle / 2-Video game / 11-Motorsport / 55-Video game console / 199-Wii / 456-Go-kart / 518-Mario Kart / \n",
"0-Games / 5-Dance / 739-Guild Wars / 839-Guild Wars 2 / \n",
"359-Harry Potter (Literary Series) / 1436-Draco Malfoy / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 179-Kitchen / 210-Cookware and bakeware / 274-Meat / \n",
"2-Video game / 309-Computer hardware / \n",
"9-Music video / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"512-Printing / 525-Printer / 619-Manufacturing / \n",
"6-Animation / 25-Toy / 524-Littlest Pet Shop / \n",
"5-Dance / 130-Gymnastics / \n",
"45-Cosmetics / \n",
"28-Fashion / 153-Dress / \n",
"12-Food / \n",
"23-Mobile phone / 29-Smartphone / 70-Driving / 223-Weather / \n",
"0-Games / 2-Video game / 55-Video game console / 251-Sonic the Hedgehog / 320-Super Smash Bros. / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"49-School / \n",
"9-Music video / 14-Guitar / \n",
"12-Food / 2138-Compost / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / \n",
"2-Video game / 561-Madden NFL / 2916-Madden NFL 12 / \n",
"0-Games / 33-Weapon / 664-CrossFire / \n",
"10-Animal / 166-Cat / 397-Organ / 502-Sky / 646-Pipe organ / \n",
"0-Games / 6-Animation / \n",
"1-Vehicle / 4-Car / 40-Road / 64-Transport / 70-Driving / 74-Truck / 212-Highway / \n",
"36-Piano / 446-Sheet music / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / \n",
"3-Concert / 7-Musician / \n",
"3-Concert / 13-Musical ensemble / 46-Choir / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"2716-The Hunger Games / \n",
"6-Animation / 15-Cartoon / 4671-Little Busters! / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 127-Xbox 360 / 181-Xbox / 297-Grand Theft Auto IV / \n",
"66-Bollywood / \n",
"93-Comedy / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 105-Motorcycling / \n",
"47-Personal computer / 1043-Router / \n",
"0-Games / 2-Video game / 55-Video game console / 116-Pokémon / 292-Handheld game console / 320-Super Smash Bros. / 335-Nintendo 3DS / 443-Wii U / 697-Super Smash Bros. for Nintendo 3DS and Wii U / \n",
"12-Food / 26-Cooking / 32-Recipe / 109-Cooking show / 239-Roasting / 542-Potato / 1701-French fries / 2266-Potato chip / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 608-Wheelie / 927-Supermoto / 4308-Husaberg / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 203-Marching band / \n",
"1-Vehicle / 4-Car / 40-Road / 64-Transport / 575-Traffic / 663-Street / 1020-Emergency vehicle / 3245-Mercedes-Benz Sprinter / \n",
"6-Animation / 15-Cartoon / 123-Naruto / 185-Dragon Ball / 244-Goku / \n",
"45-Cosmetics / 238-Nail / 300-Nail art / 304-Nail polish / 307-Hand / 324-Finger / 338-Manicure / 402-Jewellery / 823-Artificial nails / \n",
"0-Games / 2-Video game / 19-PC game / 1861-Smite / \n",
"0-Games / 6-Animation / 1676-Aikatsu! / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 1028-Devil May Cry / 1893-Devil May Cry 4 / \n",
"3-Concert / 5-Dance / 16-Performance art / 66-Bollywood / \n",
"1-Vehicle / 18-Outdoor recreation / 124-Tractor / 156-Agriculture / \n",
"6-Animation / 25-Toy / 84-Snow / 99-Christmas / 120-Winter / 126-The Walt Disney Company / 354-Egg / 417-Play-Doh / 507-Thomas the Tank Engine / 611-Kinder Surprise / 2145-Olaf / \n",
"5-Dance / 16-Performance art / \n",
"7-Musician / 38-Orchestra / \n",
"429-Website / \n",
"246-Furniture / 369-Resort / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 105-Motorcycling / 158-Tire / \n",
"5-Dance / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"325-Vampire / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"9-Music video / \n",
"12-Food / 389-Restaurant / \n",
"8-Football / \n",
"1-Vehicle / 286-Bus / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 152-Track / 868-Tram / \n",
"1726-Harlem Shake (meme) / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / 1259-Fender Telecaster / \n",
"5-Dance / \n",
"0-Games / 89-Comics / 1676-Aikatsu! / \n",
"18-Outdoor recreation / 1367-Golf club / 3140-Putter / 3264-Golf ball / \n",
"3-Concert / \n",
"14-Guitar / 24-String instrument / 96-Soldier / \n",
"14-Guitar / 100-Electric guitar / 1754-Rocksmith / 2050-Rocksmith 2014 / \n",
"1-Vehicle / 4-Car / 277-Sport utility vehicle / 403-Toyota / 1461-Electric car / 3758-Toyota RAV4 / \n",
"3-Concert / 5-Dance / 31-Disc jockey / 133-Nightclub / \n",
"66-Bollywood / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 932-Microsoft Lumia / \n",
"1970-Axe / \n",
"83-Skateboarding / 555-Longboard / 667-Longboarding / \n",
"10-Animal / 48-Pet / 71-Dog / 1603-Fox / \n",
"3-Concert / 5-Dance / \n",
"6-Animation / 1303-Fullmetal Alchemist / \n",
"928-Borderlands 2 / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 197-Trail / 608-Wheelie / \n",
"8-Football / \n",
"12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / 226-Dessert / 232-Cake / 236-Baking / 295-Chocolate / 485-Dough / 560-Sugar / 600-Flour / 1728-Chocolate cake / 2777-Mousse / \n",
"5-Dance / \n",
"1791-Stargate / \n",
"9-Music video / \n",
"95-Talent show / 130-Gymnastics / 1474-Acrobatic gymnastics / \n",
"7-Musician / 230-Brass instrument / 296-Saxophone / \n",
"0-Games / 2-Video game / 192-Battlefield / 374-Battlefield 3 / \n",
"3-Concert / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 210-Cookware and bakeware / 274-Meat / 572-Indian cuisine / 1748-Carrot / 3205-Beetroot / \n",
"1-Vehicle / 11-Motorsport / 198-Off-road vehicle / 213-Forest / 327-Mud / 565-Mud bogging / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 508-iPhone 5 / 675-iPhone 4S / 760-iPhone 5s / 1009-iPhone 3G / 1774-iPhone 5c / \n",
"0-Games / 2-Video game / 35-Minecraft / 2398-Personal armor / \n",
"28-Fashion / 785-Jeans / 1534-H&M / 2188-Coat / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 207-Exhaust system / 336-Scooter / 1036-Moped / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 131-Computer / 141-Microsoft Windows / 272-iPod / 331-iPod touch / \n",
"84-Snow / 120-Winter / 3493-Tubing / \n",
"452-Watch / 1260-Smartwatch / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 651-Stock car racing / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"0-Games / 8-Football / 65-Kick / 76-Ball / \n",
"10-Animal / 48-Pet / 71-Dog / 1377-Golden Retriever / \n",
"1-Vehicle / 40-Road / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 161-Train station / 224-Rapid transit / 663-Street / 1960-Traffic light / \n",
"2-Video game / 43-Call of Duty / 111-PlayStation 3 / 127-Xbox 360 / 302-Call of Duty 4: Modern Warfare / \n",
"110-Album / \n",
"1-Vehicle / 4-Car / 20-Trailer / 74-Truck / 796-Fire engine / \n",
"3-Concert / \n",
"5-Dance / 16-Performance art / 115-Ballet / 980-Solo dance / \n",
"0-Games / 6-Animation / 185-Dragon Ball / 241-Fighting game / 244-Goku / 418-Gohan / 1462-Dragon Ball XenoVerse / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"66-Bollywood / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / 3640-Akira Cine-Manga NeoTokyo 2019 / \n",
"36-Piano / 53-Keyboard / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / \n",
"121-Photography / 155-Camera / 566-Digital camera / 3796-Sony NEX-5 / \n",
"1-Vehicle / 4-Car / 344-Coupé / 3229-Opel Kadett / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / 361-Flamenco / 2380-Flamenco guitar / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 231-Supercar / 1400-rFactor / 4395-Sim racing / \n",
"16-Performance art / 66-Bollywood / \n",
"12-Food / 246-Furniture / 389-Restaurant / 1230-Italian food / \n",
"0-Games / 2-Video game / 55-Video game console / 586-Nintendo Entertainment System / \n",
"0-Games / 59-Winter sport / 112-Ice skating / 142-Hockey / 262-Ice / 312-Ice rink / \n",
"28-Fashion / 153-Dress / 361-Flamenco / 495-Runway / \n",
"25-Toy / 477-Transformers / \n",
"31-Disc jockey / \n",
"5-Dance / 68-Lighting / 115-Ballet / \n",
"61-Art / 276-Room / 353-Wall / \n",
"6-Animation / 15-Cartoon / 183-Manga / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 76-Ball / \n",
"45-Cosmetics / 436-Concealer / \n",
"3-Concert / \n",
"2449-Chuck E. Cheese's / \n",
"2-Video game / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / \n",
"0-Games / 2-Video game / 55-Video game console / 292-Handheld game console / 320-Super Smash Bros. / 335-Nintendo 3DS / 443-Wii U / 697-Super Smash Bros. for Nintendo 3DS and Wii U / \n",
"0-Games / 6-Animation / \n",
"5-Dance / 22-Nature / 98-Festival / \n",
"36-Piano / \n",
"264-Star Wars / 2117-Star Wars Battlefront / \n",
"8-Football / \n",
"3-Concert / \n",
"6-Animation / 126-The Walt Disney Company / 576-Walt Disney World / 3545-Pluto / \n",
"1-Vehicle / 4-Car / 299-Volkswagen Passenger Cars / 531-Hatchback / \n",
"3-Concert / \n",
"0-Games / 582-Candy / 1112-Candy Crush Saga / \n",
"60-Basketball / \n",
"0-Games / 2-Video game / 35-Minecraft / 366-Dragon / 420-The Elder Scrolls V: Skyrim / \n",
"0-Games / 1698-Darts / \n",
"5-Dance / 9-Music video / \n",
"163-Running / \n",
"0-Games / 2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / 2285-LG G3 / \n",
"1-Vehicle / 6-Animation / 136-Gardening / 585-Lawn / 741-Mower / 805-Lawn mower / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 1820-Motorola Droid / 3186-Motorola Razr / \n",
"22-Nature / 113-House / 136-Gardening / \n",
"20-Trailer / 159-World of Warcraft / 171-Warcraft / \n",
"3-Concert / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 2319-Zara / \n",
"78-Wedding / \n",
"56-Hair / 106-Hairstyle / 583-Wig / \n",
"0-Games / 2-Video game / 43-Call of Duty / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / 768-Diamond / \n",
"5-Dance / 16-Performance art / \n",
"0-Games / 25-Toy / 417-Play-Doh / 723-Pig / \n",
"28-Fashion / 415-Sewing / 880-Trousers / 1627-Shorts / \n",
"6-Animation / 15-Cartoon / 20-Trailer / 1724-Homer Simpson / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 167-Four-wheel drive / 198-Off-road vehicle / 327-Mud / 935-Off-road racing / \n",
"10-Animal / 412-Monster / 550-Dinosaur / \n",
"0-Games / 8-Football / 21-Stadium / 77-Arena / \n",
"3-Concert / 7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / 256-Tennis / \n",
"125-Television / \n",
"0-Games / 2-Video game / 55-Video game console / 292-Handheld game console / 317-PlayStation / 362-PlayStation Portable / 3980-Sega CD / \n",
"405-Braid / 457-Knitting / 546-Stitch / 653-Thread / \n",
"3-Concert / 5-Dance / 7-Musician / 13-Musical ensemble / \n",
"376-Flute / 1326-Recorder / 3344-Tin whistle / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 3649-Metroid Prime / \n",
"253-Carnival / \n",
"0-Games / 489-Board game / 799-Chess / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 169-Rallying / \n",
"920-Chair / \n",
"842-Harp / \n",
"6-Animation / 9-Music video / 15-Cartoon / \n",
"0-Games / 8-Football / 21-Stadium / 77-Arena / \n",
"56-Hair / 78-Wedding / 106-Hairstyle / 190-Bride / 405-Braid / 909-Updo / \n",
"66-Bollywood / 1524-Zee Bangla / \n",
"9-Music video / 15-Cartoon / \n",
"0-Games / 2-Video game / 90-Sports game / 3764-EA Sports UFC / 4132-Fight Night Champion / \n",
"1-Vehicle / 62-Train / 64-Transport / 94-Dashcam / 103-Rail transport / 152-Track / 161-Train station / \n",
"85-Combat / 3604-Chokehold / \n",
"3-Concert / \n",
"163-Running / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / 813-Running back / \n",
"31-Disc jockey / \n",
"804-Shopping mall / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / \n",
"1-Vehicle / 18-Outdoor recreation / 27-Motorcycle / 83-Skateboarding / 108-Skateboard / 608-Wheelie / \n",
"1721-Tetris / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 44-Drums / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / 955-Police officer / \n",
"110-Album / \n",
"159-World of Warcraft / 171-Warcraft / \n",
"5-Dance / 16-Performance art / \n",
"1-Vehicle / 4-Car / 6-Animation / 20-Trailer / 126-The Walt Disney Company / 1077-Lightning McQueen / \n",
"10-Animal / 467-Chipmunk / 846-Squirrel / \n",
"7-Musician / 14-Guitar / 24-String instrument / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"6-Animation / 10-Animal / 636-Rabbit / \n",
"315-Knife / \n",
"7-Musician / 14-Guitar / 24-String instrument / 39-Drums / 63-Acoustic guitar / \n",
"3-Concert / 7-Musician / 39-Drums / 44-Drums / 110-Album / \n",
"33-Weapon / 85-Combat / 96-Soldier / \n",
"5-Dance / 12-Food / 30-Drummer / 1465-FOX / \n",
"0-Games / 2-Video game / 10-Animal / 51-Strategy video game / \n",
"1-Vehicle / 10-Animal / 11-Motorsport / \n",
"5-Dance / 16-Performance art / 66-Bollywood / \n",
"3-Concert / 620-Barbecue / \n",
"1-Vehicle / 27-Motorcycle / \n",
"66-Bollywood / \n",
"5-Dance / \n",
"113-House / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 179-Kitchen / 232-Cake / 239-Roasting / 742-Countertop / 1018-Salmon / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 197-Trail / 220-Mountain bike / 284-Mountain biking / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 73-Ball / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / 44-Drums / \n",
"5-Dance / 227-Water / 490-Rain / \n",
"411-MapleStory / \n",
"0-Games / 8-Football / 73-Ball / 81-Athlete / 358-Rugby football / \n",
"2-Video game / 441-The Elder Scrolls / 1410-The Elder Scrolls IV: Oblivion / \n",
"14-Guitar / \n",
"10-Animal / 18-Outdoor recreation / 42-Fishing / 91-Fish / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 101-Telephone / 760-iPhone 5s / 1774-iPhone 5c / \n",
"147-River / 283-Rock / 347-Climbing / \n",
"163-Running / 206-Shoe / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"25-Toy / 55-Video game console / 4183-Tamagotchi / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / 451-Cockpit / 719-Boeing 747 / \n",
"9-Music video / \n",
"25-Toy / 626-Puzzle / 631-Cube / 740-Rubik's Cube / \n",
"1-Vehicle / 62-Train / 119-Locomotive / 507-Thomas the Tank Engine / 2332-Percy / \n",
"116-Pokémon / 186-Pokémon / 690-Pokémon X and Y / 1005-Pokémon Omega Ruby and Alpha Sapphire / \n",
"9-Music video / 125-Television / 322-Mixtape / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 1252-Dog agility / 1839-Rottweiler / \n",
"28-Fashion / \n",
"9-Music video / 31-Disc jockey / \n",
"49-School / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 94-Dashcam / \n",
"6-Animation / 15-Cartoon / \n",
"3-Concert / 16-Performance art / \n",
"113-House / 339-Architecture / \n",
"0-Games / 211-RuneScape / \n",
"0-Games / 12-Food / 389-Restaurant / \n",
"5-Dance / 16-Performance art / \n",
"6-Animation / 15-Cartoon / 49-School / \n",
"0-Games / 88-Machine / 240-Arcade game / 489-Board game / 535-Slot machine / 3235-Monopoly / 4541-Multiplayer game / \n",
"25-Toy / 140-LEGO / 462-Table / 920-Chair / \n",
"12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / 109-Cooking show / 533-Oil / 1765-Muffin / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / 397-Organ / \n",
"9-Music video / \n",
"0-Games / 3274-DragonFable / \n",
"3-Concert / 16-Performance art / 4284-Arte / \n",
"59-Winter sport / 112-Ice skating / 142-Hockey / 522-Helmet / \n",
"12-Food / 26-Cooking / 52-Dish / 58-Cuisine / 215-Eating / 301-Meal / 727-Curry / \n",
"2-Video game / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 431-Call of Duty: Advanced Warfare / 1843-Ham / \n",
"497-Macintosh / 3233-Screencast / \n",
"1-Vehicle / 4-Car / \n",
"122-Weight training / 204-Gym / 214-Muscle / 1055-Back / \n",
"9-Music video / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 1124-Yamaha YZF-R1 / \n",
"47-Personal computer / 145-Tablet computer / 189-iPad / 2262-E-reader / \n",
"3-Concert / 68-Lighting / \n",
"527-Battery / 1642-Cigarette / \n",
"1-Vehicle / 42-Fishing / 87-Boat / 498-Motorboat / 579-Yacht / 753-Sailboat / \n",
"3-Concert / \n",
"3-Concert / 38-Orchestra / \n",
"4214-Spike / \n",
"0-Games / 1-Vehicle / 2-Video game / 2101-Ace Combat / \n",
"113-House / 179-Kitchen / 246-Furniture / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 152-Track / 428-Rail transport modelling / \n",
"93-Comedy / 364-Sitcom / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 150-Violin / 639-Fiddle / \n",
"3-Concert / 16-Performance art / \n",
"0-Games / 326-Metin2 / \n",
"7-Musician / 46-Choir / 126-The Walt Disney Company / 153-Dress / 576-Walt Disney World / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 111-PlayStation 3 / 127-Xbox 360 / 181-Xbox / 2621-GameTrailers / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 111-PlayStation 3 / 352-Need for Speed / 1419-GameSpot / 2219-Need for Speed: Shift / \n",
"31-Disc jockey / \n",
"9-Music video / 322-Mixtape / \n",
"293-Slide show / \n",
"20-Trailer / 359-Harry Potter (Literary Series) / \n",
"12-Food / 26-Cooking / 32-Recipe / 226-Dessert / 534-Milk / 770-Banana / 1283-Pancake / \n",
"8-Football / \n",
"37-Gadget / 47-Personal computer / 131-Computer / 141-Microsoft Windows / 145-Tablet computer / \n",
"36-Piano / 88-Machine / 400-Woodturning / 673-Saw / 1700-Lumber / 2874-Sawmill / 3789-Circular saw / \n",
"13-Musical ensemble / \n",
"3-Concert / 7-Musician / \n",
"144-Amusement park / 329-Swimming pool / 1053-Water park / \n",
"47-Personal computer / 228-Laptop / 513-Computer keyboard / \n",
"6-Animation / 15-Cartoon / \n",
"3-Concert / \n",
"25-Toy / 417-Play-Doh / 3571-Lollipop / \n",
"2-Video game / 43-Call of Duty / 111-PlayStation 3 / 127-Xbox 360 / 134-Call of Duty: Black Ops / \n",
"0-Games / 1605-Second Life / \n",
"66-Bollywood / \n",
"3-Concert / 7-Musician / 14-Guitar / 16-Performance art / \n",
"5-Dance / \n",
"514-Tekken / \n",
"0-Games / 159-World of Warcraft / 171-Warcraft / 3615-Monk / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 154-Wheel / 158-Tire / 270-Sedan / 1455-Cadillac / 3681-Cadillac CTS / \n",
"3-Concert / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / 297-Grand Theft Auto IV / 332-Mercedes-Benz / \n",
"1-Vehicle / 4-Car / \n",
"122-Weight training / \n",
"9-Music video / \n",
"72-iPhone / \n",
"0-Games / 51-Strategy video game / 165-League of Legends / \n",
"6-Animation / 15-Cartoon / \n",
"5-Dance / \n",
"1-Vehicle / 4-Car / 403-Toyota / 3758-Toyota RAV4 / \n",
"3-Concert / 7-Musician / \n",
"455-Unidentified flying object / 822-Crane / 1668-Claw crane / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 192-Battlefield / 374-Battlefield 3 / \n",
"253-Carnival / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 553-Sony Xperia / \n",
"170-Home improvement / 276-Room / 652-Bathroom / 1011-Tile / 1113-Toilet / 1272-Cabinetry / 2153-Sink / \n",
"122-Weight training / 204-Gym / \n",
"3-Concert / 16-Performance art / 594-Music festival / \n",
"0-Games / 8-Football / 21-Stadium / 77-Arena / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 108-Skateboard / 305-Ski / 336-Scooter / 667-Longboarding / 703-Roller skating / \n",
"0-Games / 2-Video game / 51-Strategy video game / 1072-Total War / 1882-Warrior / 2810-Total War: Rome II / \n",
"6-Animation / 25-Toy / 140-LEGO / \n",
"0-Games / 20-Trailer / 4304-Civilization V / \n",
"2811-Llanero / \n",
"1-Vehicle / 4-Car / 170-Home improvement / 519-Recreational vehicle / \n",
"3-Concert / 13-Musical ensemble / \n",
"0-Games / 2-Video game / 43-Call of Duty / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / 1248-Call of Duty: Zombies / \n",
"75-Wrestling / 85-Combat / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / 224-Rapid transit / \n",
"45-Cosmetics / 173-Eye shadow / 195-Mascara / 196-Eye / 218-Eyelash / 3106-Eyelash extensions / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / 2610-Rosary / \n",
"9-Music video / \n",
"6-Animation / 15-Cartoon / 643-Hero / \n",
"14-Guitar / 96-Soldier / \n",
"0-Games / 2-Video game / 43-Call of Duty / 138-Call of Duty: Black Ops II / \n",
"122-Weight training / \n",
"527-Battery / 2484-Electronic circuit / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / 858-Carl Johnson / \n",
"5-Dance / 95-Talent show / 130-Gymnastics / 191-Cheerleading / 892-Aerobics / 2228-Sport aerobics / \n",
"33-Weapon / 217-Hunting / \n",
"0-Games / 2-Video game / 257-Counter-Strike / 315-Knife / \n",
"47-Personal computer / 345-Loudspeaker / 414-Computer monitor / \n",
"5-Dance / 49-School / 77-Arena / 95-Talent show / 130-Gymnastics / 191-Cheerleading / 306-High school / \n",
"5-Dance / 13-Musical ensemble / 78-Wedding / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 47-Personal computer / 101-Telephone / 145-Tablet computer / 621-Asus / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 96-Soldier / 704-Call of Duty: World at War / \n",
"0-Games / 75-Wrestling / \n",
"10-Animal / 25-Toy / 166-Cat / 524-Littlest Pet Shop / 749-Figurine / \n",
"6-Animation / \n",
"5-Dance / 16-Performance art / 115-Ballet / 430-Ballroom dance / \n",
"3-Concert / 68-Lighting / \n",
"429-Website / 1601-Social media / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 234-Ford / 473-Ford Mustang / 647-Muscle car / 1877-Shelby Mustang / \n",
"99-Christmas / \n",
"696-T-shirt / \n",
"351-Jumping / 529-Squat / \n",
"0-Games / 2-Video game / 35-Minecraft / 102-Building / 1067-Lock / \n",
"0-Games / 60-Basketball / 81-Athlete / 168-Basketball moves / 255-Slam dunk / \n",
"1-Vehicle / 4-Car / \n",
"696-T-shirt / 793-Shirt / 2693-Puma SE / \n",
"1-Vehicle / 27-Motorcycle / 154-Wheel / \n",
"33-Weapon / 1873-Crossbow / \n",
"121-Photography / \n",
"1-Vehicle / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 197-Trail / 220-Mountain bike / 284-Mountain biking / \n",
"238-Nail / 300-Nail art / 304-Nail polish / 307-Hand / 324-Finger / 338-Manicure / \n",
"93-Comedy / 355-Comedian / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 70-Driving / 2286-Grand Tourer Injection / 2826-Suzuki Swift / \n",
"1-Vehicle / 4-Car / 519-Recreational vehicle / 707-Roof / \n",
"9-Music video / 78-Wedding / 190-Bride / \n",
"3-Concert / 68-Lighting / \n",
"242-Paper / 370-Origami / \n",
"1-Vehicle / 27-Motorcycle / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 70-Driving / 2249-Driver / 3365-Driver: San Francisco / \n",
"25-Toy / 140-LEGO / 249-Heavy equipment / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 4236-Biryani / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 57-Race track / 378-Drag racing / \n",
"18-Outdoor recreation / 42-Fishing / 164-Beach / 222-Surfing / 233-Ocean / 423-Coast / \n",
"22-Nature / 1235-Rainbow / \n",
"5-Dance / \n",
"0-Games / 637-Five Nights at Freddy's / \n",
"28-Fashion / 206-Shoe / 333-Nike; Inc. / 368-Sneakers / \n",
"31-Disc jockey / \n",
"22-Nature / 490-Rain / 998-Landscape / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 42-Fishing / 147-River / 478-Fishing rod / 825-Stream / 895-Trout / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 531-Hatchback / 618-Fiat Automobiles / 3127-Fiat Uno / \n",
"2449-Chuck E. Cheese's / \n",
"0-Games / 2-Video game / 229-Halo / 470-Halo 3 / 2035-Forge / \n",
"1-Vehicle / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 197-Trail / 213-Forest / 220-Mountain bike / 284-Mountain biking / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 192-Battlefield / 374-Battlefield 3 / 383-Battlefield 4 / \n",
"10-Animal / 20-Trailer / 80-Horse / \n",
"18-Outdoor recreation / 22-Nature / 222-Surfing / 233-Ocean / 672-Bodyboarding / \n",
"75-Wrestling / \n",
"0-Games / 2-Video game / 1959-Phantasy Star / 2142-Sega Saturn / \n",
"0-Games / 2-Video game / 55-Video game console / 413-PlayStation 2 / \n",
"5-Dance / 115-Ballet / \n",
"110-Album / \n",
"130-Gymnastics / \n",
"12-Food / 26-Cooking / 52-Dish / 58-Cuisine / 2959-Wok / \n",
"0-Games / 2-Video game / 19-PC game / 20-Trailer / \n",
"7-Musician / 13-Musical ensemble / \n",
"10-Animal / \n",
"3-Concert / 77-Arena / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 294-Batman / 838-Batman: Arkham / 1660-Batman: Arkham Asylum / 1881-Arkham Asylum / \n",
"989-Warface / \n",
"6-Animation / 15-Cartoon / \n",
"130-Gymnastics / 1255-Ribbon / 1451-Ribbon / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / 379-Touhou Project / 592-Synthesia / \n",
"22-Nature / 86-Plant / 136-Gardening / 137-Tree / \n",
"102-Building / 2347-Marble / \n",
"12-Food / 18-Outdoor recreation / 26-Cooking / 32-Recipe / 136-Gardening / 395-Camping / 968-Pie / \n",
"9-Music video / \n",
"242-Paper / 315-Knife / 512-Printing / 525-Printer / \n",
"5-Dance / 16-Performance art / \n",
"0-Games / 116-Pokémon / 186-Pokémon / \n",
"860-Christian Church / 3783-Blazer / \n",
"59-Winter sport / 112-Ice skating / 142-Hockey / 262-Ice / 312-Ice rink / \n",
"1-Vehicle / 4-Car / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"0-Games / 19-PC game / 51-Strategy video game / 165-League of Legends / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"12-Food / 25-Toy / 26-Cooking / 1567-Cook / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"95-Talent show / 130-Gymnastics / 1417-Balance beam / \n",
"2-Video game / 989-Warface / \n",
"10-Animal / 48-Pet / 162-Medicine / 166-Cat / \n",
"93-Comedy / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / 44-Drums / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / 2738-Fender Jazz Bass / \n",
"3-Concert / 5-Dance / 99-Christmas / \n",
"0-Games / 8-Football / 81-Athlete / 358-Rugby football / \n",
"3-Concert / 68-Lighting / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / \n",
"5-Dance / \n",
"6-Animation / 15-Cartoon / 185-Dragon Ball / 244-Goku / 418-Gohan / 1244-Trunks / 1304-Cell / \n",
"242-Paper / 313-Book / 415-Sewing / 2454-Notebook / \n",
"20-Trailer / \n",
"261-Tool / 673-Saw / 1033-Drill / \n",
"3-Concert / 7-Musician / 219-Accordion / 545-Quartet (ensemble) / \n",
"624-Number / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / \n",
"31-Disc jockey / \n",
"0-Games / 2-Video game / 1007-Portal / 1069-Easter egg / 1274-Portal 2 / 3555-Valve Corporation / \n",
"10-Animal / 71-Dog / \n",
"1249-Rainbow Loom / 1449-Rubber band / \n",
"3-Concert / \n",
"3-Concert / 7-Musician / \n",
"358-Rugby football / \n",
"1-Vehicle / 27-Motorcycle / 263-Light / \n",
"1-Vehicle / 11-Motorsport / 64-Transport / 74-Truck / 822-Crane / \n",
"6-Animation / 15-Cartoon / 123-Naruto / \n",
"3-Concert / 5-Dance / 16-Performance art / 1572-Husband / \n",
"259-Fire / 371-Firefighter / \n",
"0-Games / 35-Minecraft / \n",
"1338-Devil / 2793-Disguise / \n",
"10-Animal / 48-Pet / 1147-Hamster / \n",
"1783-Roller skates / \n",
"6-Animation / 15-Cartoon / 801-Inuyasha / \n",
"1-Vehicle / 4-Car / 912-Silver / \n",
"5-Dance / 16-Performance art / 980-Solo dance / \n",
"1-Vehicle / 12-Food / 1054-Vacuum cleaner / \n",
"2128-Pollution / \n",
"28-Fashion / 56-Hair / 106-Hairstyle / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 74-Truck / 349-Chevrolet / 375-Pickup truck / 406-Chevrolet / \n",
"1-Vehicle / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 197-Trail / 220-Mountain bike / 284-Mountain biking / \n",
"0-Games / 8-Football / 54-Highlight film / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 266-Helicopter / 383-Battlefield 4 / \n",
"780-Fat / \n",
"5-Dance / 20-Trailer / 78-Wedding / \n",
"6-Animation / 15-Cartoon / 1946-Total Drama Island / \n",
"31-Disc jockey / \n",
"1-Vehicle / 50-Aircraft / 1409-Missile / 4341-Anti-aircraft warfare / \n",
"9-Music video / \n",
"31-Disc jockey / 322-Mixtape / \n",
"1883-Gothic / 2515-Gothic / \n",
"45-Cosmetics / 195-Mascara / \n",
"0-Games / 60-Basketball / \n",
"34-Action-adventure game / 111-PlayStation 3 / 1125-God of War / 3927-God of War: Ascension / \n",
"0-Games / 2-Video game / 116-Pokémon / 186-Pokémon / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"9-Music video / 78-Wedding / 153-Dress / 190-Bride / \n",
"5-Dance / \n",
"79-American football / \n",
"3-Concert / 13-Musical ensemble / 46-Choir / 49-School / \n",
"5-Dance / \n",
"1442-Tabla / \n",
"47-Personal computer / 957-Desktop computer / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 301-Meal / 597-Soup / \n",
"25-Toy / 75-Wrestling / 435-Action figure / 1844-WWE action figures / \n",
"223-Weather / \n",
"341-Sketch comedy / \n",
"0-Games / \n",
"2-Video game / 55-Video game console / 413-PlayStation 2 / \n",
"20-Trailer / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 413-PlayStation 2 / 886-James Bond / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 76-Ball / 81-Athlete / \n",
"99-Christmas / 729-Christmas decoration / 964-Christmas tree / \n",
"17-Racing / 67-Cycling / 69-Bicycle / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 105-Motorcycling / \n",
"12-Food / 1596-Cinnamon / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 36-Piano / 38-Orchestra / \n",
"1-Vehicle / 4-Car / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 30-Drummer / 39-Drums / 44-Drums / \n",
"341-Sketch comedy / 364-Sitcom / 463-Cricket / \n",
"322-Mixtape / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 30-Drummer / \n",
"75-Wrestling / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 169-Rallying / 1122-Mitsubishi Lancer Evolution / \n",
"1-Vehicle / \n",
"298-Television advertisement / 4145-Tape recorder / \n",
"1-Vehicle / 4-Car / 40-Road / 64-Transport / 70-Driving / 94-Dashcam / 212-Highway / \n",
"0-Games / 6-Animation / 15-Cartoon / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / \n",
"420-The Elder Scrolls V: Skyrim / \n",
"102-Building / 113-House / 433-Construction / \n",
"0-Games / 2-Video game / 34-Action-adventure game / \n",
"0-Games / 8-Football / 81-Athlete / \n",
"1-Vehicle / 50-Aircraft / 139-Airport / 151-Landing / 175-Airliner / 188-Airline / 472-Boeing 737 / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / \n",
"10-Animal / 350-Circus / \n",
"5-Dance / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 94-Dashcam / 212-Highway / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 73-Ball / \n",
"56-Hair / 583-Wig / 607-Lace / 871-Hair coloring / \n",
"260-Prayer / 616-The Bible / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / \n",
"84-Snow / 120-Winter / \n",
"0-Games / 9-Music video / 326-Metin2 / \n",
"117-Boxing / \n",
"13-Musical ensemble / 110-Album / \n",
"6-Animation / 35-Minecraft / \n",
"0-Games / 2-Video game / 1745-Assassin's Creed: Revelations / \n",
"1-Vehicle / 4-Car / 1067-Lock / \n",
"89-Comics / 269-Comic book / 313-Book / \n",
"1546-Line / \n",
"42-Fishing / 510-Fishing lure / 902-Fishing bait / 1836-Fly tying / \n",
"0-Games / 59-Winter sport / 112-Ice skating / 142-Hockey / 312-Ice rink / \n",
"5-Dance / 9-Music video / 1443-Dance studio / \n",
"1-Vehicle / 10-Animal / 18-Outdoor recreation / 42-Fishing / 87-Boat / 91-Fish / 268-Recreational fishing / 698-Fisherman / 1471-Tuna / \n",
"1-Vehicle / 259-Fire / 371-Firefighter / \n",
"3-Concert / 1684-Hillsong Church / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 171-Warcraft / 1691-Warcraft III: Reign of Chaos / 2549-Warcraft III: The Frozen Throne / \n",
"725-Human / 1264-Head / \n",
"36-Piano / \n",
"102-Building / 627-Clock / \n",
"0-Games / 2-Video game / 89-Comics / 185-Dragon Ball / 244-Goku / 876-Freeza / 1462-Dragon Ball XenoVerse / \n",
"2507-Macramé / 3063-Friendship bracelet / \n",
"31-Disc jockey / \n",
"25-Toy / 387-Spider-Man / 435-Action figure / 4288-Carnage / \n",
"25-Toy / 89-Comics / 185-Dragon Ball / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 235-Lipstick / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 2381-Menu / \n",
"1-Vehicle / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 197-Trail / 208-Wood / 220-Mountain bike / 284-Mountain biking / \n",
"9-Music video / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / \n",
"10-Animal / 56-Hair / \n",
"1430-Pull-up / \n",
"950-Blu-ray disc / \n",
"1-Vehicle / 271-Tank / 273-News program / \n",
"1-Vehicle / 4-Car / 40-Road / 64-Transport / 74-Truck / \n",
"10-Animal / 48-Pet / 71-Dog / 2211-Fur / 2735-Raccoon / \n",
"0-Games / 756-Tibia / \n",
"238-Nail / 304-Nail polish / 324-Finger / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"6-Animation / 15-Cartoon / 2710-Shikamaru Nara / 3702-Temari / \n",
"110-Album / \n",
"6-Animation / 200-Doll / 439-Barbie / \n",
"1-Vehicle / 4-Car / 744-Window / 918-Brake / 4327-Window film / \n",
"98-Festival / \n",
"13-Musical ensemble / \n",
"4357-Nezha / \n",
"5-Dance / 49-School / 95-Talent show / \n",
"8-Football / 14-Guitar / \n",
"25-Toy / 1154-Anpanman / \n",
"12-Food / 26-Cooking / 32-Recipe / 770-Banana / \n",
"12-Food / 26-Cooking / 32-Recipe / 236-Baking / 485-Dough / 578-Cookie / \n",
"15-Cartoon / 116-Pokémon / 1104-Ash Ketchum / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 167-Four-wheel drive / 198-Off-road vehicle / 213-Forest / 327-Mud / 565-Mud bogging / \n",
"217-Hunting / 718-Duck / 1914-Waterfowl hunting / \n",
"14-Guitar / 100-Electric guitar / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"130-Gymnastics / \n",
"6-Animation / 20-Trailer / \n",
"5-Dance / 16-Performance art / \n",
"22-Nature / 84-Snow / 120-Winter / 918-Brake / 2058-Snowman / \n",
"12-Food / 232-Cake / 795-Cupcake / 1707-Microwave oven / \n",
"211-RuneScape / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 105-Motorcycling / \n",
"99-Christmas / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 121-Photography / \n",
"276-Room / 476-Textile / \n",
"1386-ABS-CBN / 1849-ABS-CBN News and Current Affairs / \n",
"5-Dance / 115-Ballet / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / 318-Church / 397-Organ / 870-Chapel / \n",
"462-Table / 544-Table tennis / 1615-Pong / \n",
"113-House / 200-Doll / 1842-Dollhouse / 2743-Wallpaper / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 685-Luigi / 1638-Soulcalibur / 2033-Princess Peach / 2877-Soulcalibur IV / 4628-Soulcalibur III / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 111-PlayStation 3 / 138-Call of Duty: Black Ops II / \n",
"8-Football / \n",
"0-Games / 35-Minecraft / \n",
"1-Vehicle / 4-Car / 40-Road / 64-Transport / 70-Driving / 94-Dashcam / \n",
"0-Games / 19-PC game / 211-RuneScape / \n",
"3-Concert / 8-Football / 21-Stadium / 77-Arena / \n",
"0-Games / 2-Video game / 55-Video game console / 317-PlayStation / 682-PlayStation Vita / \n",
"219-Accordion / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"66-Bollywood / \n",
"126-The Walt Disney Company / 369-Resort / 574-Elevator / 576-Walt Disney World / 1746-Otis Elevator Company / \n",
"0-Games / 2-Video game / 420-The Elder Scrolls V: Skyrim / 441-The Elder Scrolls / \n",
"10-Animal / 80-Horse / 247-Stallion / 404-Livestock / 438-Mare / 1871-Foal / \n",
"286-Bus / 3694-Burton Snowboards / \n",
"122-Weight training / 954-Dreadlocks / \n",
"2-Video game / 34-Action-adventure game / 362-PlayStation Portable / 1125-God of War / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 231-Supercar / 1056-Porsche 911 / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 291-Rouge / \n",
"1-Vehicle / 74-Truck / \n",
"10-Animal / 80-Horse / 261-Tool / 1995-Hammer / 2035-Forge / \n",
"5-Dance / 16-Performance art / \n",
"1-Vehicle / 27-Motorcycle / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 1631-PlanetSide 2 / 2026-PlanetSide / \n",
"10-Animal / 22-Nature / 86-Plant / 258-Aquarium / 588-Reptile / \n",
"66-Bollywood / \n",
"14-Guitar / 63-Acoustic guitar / 1231-Mandolin / \n",
"14-Guitar / 24-String instrument / \n",
"40-Road / 492-Floor / \n",
"983-Lexus / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / 265-Electronic keyboard / 592-Synthesia / \n",
"10-Animal / 86-Plant / 136-Gardening / 174-Farm / 279-Garden / 363-Soil / 2138-Compost / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 113-House / 351-Jumping / 585-Lawn / \n",
"0-Games / 51-Strategy video game / \n",
"1-Vehicle / 18-Outdoor recreation / 87-Boat / 147-River / 225-Lake / 949-Rowing / 1090-Canoe / 1305-Canoeing / \n",
"1076-Wakeboarding / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 111-PlayStation 3 / 127-Xbox 360 / 216-Call of Duty: Modern Warfare 3 / \n",
"12-Food / 88-Machine / \n",
"5-Dance / 16-Performance art / \n",
"5-Dance / 31-Disc jockey / 98-Festival / \n",
"28-Fashion / \n",
"205-Newscaster / 2762-CBS News / \n",
"130-Gymnastics / \n",
"5-Dance / \n",
"1035-Bank / \n",
"0-Games / 2-Video game / 6-Animation / 89-Comics / 123-Naruto / 591-Naruto: Ultimate Ninja / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / 265-Electronic keyboard / \n",
"28-Fashion / 206-Shoe / 333-Nike; Inc. / 368-Sneakers / \n",
"9-Music video / \n",
"46-Choir / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 22-Nature / 86-Plant / 154-Wheel / 158-Tire / 167-Four-wheel drive / 198-Off-road vehicle / 327-Mud / 360-Jeep / 565-Mud bogging / 1406-Jeep Cherokee / 2925-Jeep Cherokee (XJ) / \n",
"1-Vehicle / 4-Car / 227-Water / \n",
"0-Games / 1-Vehicle / 2-Video game / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"18-Outdoor recreation / 22-Nature / 347-Climbing / 828-Rock climbing / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 289-Dressage / 1434-Saddle / \n",
"36-Piano / \n",
"9-Music video / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"31-Disc jockey / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 670-Banjo / \n",
"0-Games / 2-Video game / \n",
"766-President of the United States / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"1-Vehicle / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 84-Snow / 220-Mountain bike / 284-Mountain biking / 669-Extreme sport / \n",
"0-Games / 6-Animation / 15-Cartoon / 35-Minecraft / 2312-SpongeBob SquarePants / \n",
"6-Animation / 10-Animal / 15-Cartoon / 440-Lion / 1156-Simba / \n",
"6-Animation / \n",
"93-Comedy / \n",
"0-Games / 562-Pitcher / \n",
"504-Web page / 819-World Wide Web / \n",
"76-Ball / 772-Bowling / 930-Ten-pin bowling / 1217-Bowling ball / \n",
"23-Mobile phone / 29-Smartphone / \n",
"232-Cake / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"0-Games / 10-Animal / 48-Pet / \n",
"6-Animation / 15-Cartoon / 1617-Fungus / \n",
"273-News program / \n",
"12-Food / 26-Cooking / 122-Weight training / 215-Eating / 301-Meal / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 40-Road / 94-Dashcam / 105-Motorcycling / \n",
"3-Concert / 5-Dance / 13-Musical ensemble / \n",
"0-Games / 8-Football / \n",
"0-Games / 326-Metin2 / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 184-Vegetable / 1295-Stir frying / 2397-Cabbage / 2614-Lettuce / \n",
"3-Concert / 7-Musician / \n",
"31-Disc jockey / \n",
"6-Animation / 25-Toy / 298-Television advertisement / 507-Thomas the Tank Engine / 2332-Percy / \n",
"2-Video game / 55-Video game console / 292-Handheld game console / 320-Super Smash Bros. / 782-Super Smash Bros. Melee / \n",
"0-Games / 2-Video game / 33-Weapon / 734-DayZ / 1216-DayZ / \n",
"66-Bollywood / \n",
"1-Vehicle / 50-Aircraft / 102-Building / 451-Cockpit / 472-Boeing 737 / \n",
"1-Vehicle / 11-Motorsport / \n",
"1-Vehicle / 50-Aircraft / 129-Aviation / 388-Fighter aircraft / 840-Museum / \n",
"5-Dance / \n",
"1-Vehicle / 158-Tire / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 38-Orchestra / \n",
"0-Games / 241-Fighting game / 764-The King of Fighters / 2354-The King of Fighters 2002 / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 2265-Subway Surfers / \n",
"18-Outdoor recreation / 33-Weapon / 137-Tree / 213-Forest / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 74-Truck / 252-Tractor pulling / 375-Pickup truck / 854-Diesel engine / \n",
"5-Dance / 16-Performance art / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 92-Radio-controlled model / 129-Aviation / 149-Model aircraft / 157-Jet aircraft / 160-Radio-controlled aircraft / 502-Sky / \n",
"28-Fashion / 78-Wedding / 153-Dress / 190-Bride / 421-Gown / 495-Runway / 590-Wedding dress / \n",
"3-Concert / \n",
"1151-Temple / \n",
"18-Outdoor recreation / 144-Amusement park / 321-Roller coaster / 334-Amusement ride / \n",
"93-Comedy / \n",
"0-Games / 530-Cue sports / 568-Pool / 994-Cue stick / 1364-Nine-ball / \n",
"3-Concert / 5-Dance / \n",
"79-American football / \n",
"1-Vehicle / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 520-Bicycle frame / 570-Road bicycle racing / 645-Road bicycle / \n",
"1-Vehicle / 4-Car / 84-Snow / 120-Winter / 245-BMW / 1145-Winter storm / \n",
"1-Vehicle / 4-Car / 74-Truck / 124-Tractor / 158-Tire / 167-Four-wheel drive / 249-Heavy equipment / 327-Mud / \n",
"20-Trailer / \n",
"0-Games / 19-PC game / 35-Minecraft / 48-Pet / 366-Dragon / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 76-Ball / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 403-Toyota / 3416-Toyota MR2 / \n",
"47-Personal computer / 145-Tablet computer / 189-iPad / \n",
"66-Bollywood / \n",
"0-Games / 2-Video game / 47-Personal computer / 131-Computer / 957-Desktop computer / \n",
"309-Computer hardware / 908-Video card / \n",
"3-Concert / \n",
"3-Concert / 16-Performance art / 95-Talent show / \n",
"0-Games / 2-Video game / 19-PC game / 1392-Injustice: Gods Among Us / \n",
"28-Fashion / \n",
"6-Animation / 849-Tales / \n",
"1-Vehicle / 62-Train / 67-Cycling / 69-Bicycle / \n",
"0-Games / 35-Minecraft / 464-First-person Shooter / \n",
"1-Vehicle / 11-Motorsport / \n",
"9-Music video / \n",
"0-Games / 60-Basketball / 142-Hockey / \n",
"14-Guitar / 100-Electric guitar / 1103-Gibson Les Paul / \n",
"1-Vehicle / 87-Boat / 579-Yacht / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / \n",
"12-Food / 1151-Temple / 3039-Wat / \n",
"15-Cartoon / \n",
"144-Amusement park / 321-Roller coaster / 334-Amusement ride / 1293-RollerCoaster Tycoon 3 / \n",
"79-American football / \n",
"8-Football / \n",
"5-Dance / 490-Rain / \n",
"5-Dance / 16-Performance art / 115-Ballet / 191-Cheerleading / \n",
"1-Vehicle / 154-Wheel / 249-Heavy equipment / 1093-Loader / \n",
"237-Final Fantasy / 1706-Final Fantasy X-2 / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 25-Toy / 92-Radio-controlled model / 202-Radio-controlled car / 926-Traxxas / \n",
"0-Games / 2-Video game / 19-PC game / 229-Halo / 470-Halo 3 / \n",
"8-Football / 79-American football / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 685-Luigi / 3506-Luigi's Mansion / \n",
"59-Winter sport / 84-Snow / 177-Skiing / 305-Ski / 789-Alpine skiing / 1830-Downhill / \n",
"2-Video game / 43-Call of Duty / 111-PlayStation 3 / 127-Xbox 360 / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / 181-Xbox / 194-Call of Duty: Modern Warfare 2 / 431-Call of Duty: Advanced Warfare / \n",
"10-Animal / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 320-Super Smash Bros. / 782-Super Smash Bros. Melee / 2165-Ganon / \n",
"22-Nature / 259-Fire / 419-Earth / 1058-Volcano / \n",
"18-Outdoor recreation / 283-Rock / 810-Village / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 952-Chevrolet Corvette / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"84-Snow / 120-Winter / 1145-Winter storm / \n",
"56-Hair / 106-Hairstyle / 607-Lace / 1886-Bob cut / \n",
"6-Animation / 61-Art / 97-Drawing / 148-Painting / 528-Pencil / 628-Portrait / \n",
"9-Music video / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"743-Edward Cullen / \n",
"0-Games / 613-Bar / 3538-S.T.A.L.K.E.R.: Call of Pripyat / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / 1127-Drum stick / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 2097-Dachshund / \n",
"5-Dance / 9-Music video / \n",
"12-Food / 1596-Cinnamon / \n",
"99-Christmas / 729-Christmas decoration / 964-Christmas tree / \n",
"1-Vehicle / 50-Aircraft / 388-Fighter aircraft / 1483-United States Air Force / \n",
"0-Games / 33-Weapon / 257-Counter-Strike / \n",
"3-Concert / 5-Dance / 3804-Music of Eritrea / \n",
"46-Choir / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"31-Disc jockey / \n",
"8-Football / 79-American football / \n",
"1-Vehicle / 4-Car / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 94-Dashcam / 378-Drag racing / \n",
"219-Accordion / \n",
"3-Concert / 16-Performance art / \n",
"0-Games / 1876-Beatmania IIDX / \n",
"14-Guitar / 25-Toy / 1154-Anpanman / \n",
"9-Music video / \n",
"0-Games / 8-Football / 90-Sports game / 180-FIFA 15 / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 378-Drag racing / \n",
"6-Animation / 15-Cartoon / \n",
"3-Concert / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / \n",
"3-Concert / 5-Dance / \n",
"8-Football / \n",
"1-Vehicle / 83-Skateboarding / 108-Skateboard / 555-Longboard / \n",
"1-Vehicle / 11-Motorsport / 20-Trailer / 27-Motorcycle / \n",
"0-Games / 2-Video game / 637-Five Nights at Freddy's / 1027-Terraria / \n",
"8-Football / \n",
"1142-Laundry / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 43-Call of Duty / 134-Call of Duty: Black Ops / \n",
"0-Games / 60-Basketball / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 70-Driving / \n",
"1-Vehicle / 18-Outdoor recreation / 42-Fishing / \n",
"8-Football / \n",
"61-Art / 479-Graffiti / 2853-Aerosol paint / \n",
"7-Musician / 219-Accordion / 655-Diatonic button accordion / \n",
"7-Musician / 9-Music video / 14-Guitar / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 197-Trail / 327-Mud / 491-Enduro / \n",
"18-Outdoor recreation / 22-Nature / 86-Plant / 137-Tree / 722-Waterfall / \n",
"12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / 239-Roasting / 274-Meat / 391-Chicken meat / 589-Grilling / 620-Barbecue / 1085-Barbecue grill / \n",
"10-Animal / 22-Nature / 137-Tree / 213-Forest / 1108-Mushroom / 1617-Fungus / \n",
"12-Food / \n",
"0-Games / 6-Animation / 15-Cartoon / 877-Hatsune Miku: Project DIVA / \n",
"2649-Nativity scene / \n",
"3-Concert / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"3-Concert / \n",
"1-Vehicle / 11-Motorsport / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 278-GoPro / 393-Unmanned aerial vehicle / \n",
"18-Outdoor recreation / 94-Dashcam / 144-Amusement park / 278-GoPro / 321-Roller coaster / 334-Amusement ride / \n",
"2273-Polo / \n",
"0-Games / 2-Video game / 20-Trailer / 34-Action-adventure game / 111-PlayStation 3 / 127-Xbox 360 / 380-Assassin's Creed / 1048-Assassin's Creed / 1140-Assassin's Creed III / 1435-Ezio Auditore da Firenze / 1745-Assassin's Creed: Revelations / 2554-F.E.A.R. / \n",
"12-Food / 968-Pie / \n",
"1-Vehicle / 8-Football / \n",
"1-Vehicle / 50-Aircraft / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 266-Helicopter / 392-Radio-controlled helicopter / 1257-Turbine / \n",
"5-Dance / \n",
"1-Vehicle / 4-Car / 41-Sports car / 154-Wheel / 245-BMW / 493-Rim / \n",
"5-Dance / 16-Performance art / \n",
"1-Vehicle / 4-Car / 25-Toy / 200-Doll / 217-Hunting / 269-Comic book / 290-Pony / 343-My Little Pony / \n",
"0-Games / 2-Video game / 51-Strategy video game / 603-StarCraft II: Wings of Liberty / 4353-Sarah Kerrigan / \n",
"1-Vehicle / 4-Car / 270-Sedan / 864-Speedometer / 983-Lexus / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 391-Chicken meat / 424-Chicken / 500-Rice / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / 471-Wing / \n",
"0-Games / 8-Football / 21-Stadium / 77-Arena / \n",
"2424-Greeting card / \n",
"18-Outdoor recreation / 117-Boxing / \n",
"78-Wedding / \n",
"35-Minecraft / 1007-Portal / \n",
"30-Drummer / 39-Drums / 44-Drums / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / \n",
"6-Animation / 15-Cartoon / 1534-H&M / \n",
"8-Football / 358-Rugby football / \n",
"0-Games / 43-Call of Duty / 138-Call of Duty: Black Ops II / \n",
"121-Photography / \n",
"3-Concert / 7-Musician / \n",
"2-Video game / 55-Video game console / 111-PlayStation 3 / \n",
"810-Village / \n",
"293-Slide show / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"1-Vehicle / 4-Car / 94-Dashcam / 915-Town / \n",
"0-Games / 1000-AdventureQuest Worlds / \n",
"9-Music video / \n",
"12-Food / 533-Oil / 640-Bottle / 1769-Olive / 3027-Olive oil / \n",
"30-Drummer / 39-Drums / 44-Drums / \n",
"60-Basketball / \n",
"14-Guitar / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 291-Rouge / \n",
"9-Music video / \n",
"28-Fashion / 4200-London Fashion Week / \n",
"7-Musician / 14-Guitar / 24-String instrument / 219-Accordion / 2744-Bajo sexto / \n",
"0-Games / 2-Video game / 19-PC game / 464-First-person Shooter / 2460-Unreal / 3624-Quake III Arena / 3906-Unreal Tournament / \n",
"1-Vehicle / 4-Car / 124-Tractor / 156-Agriculture / 174-Farm / 523-Cattle / \n",
"1-Vehicle / 18-Outdoor recreation / 59-Winter sport / 84-Snow / 120-Winter / 610-Snowmobile / 1201-Ski-Doo / \n",
"5-Dance / 16-Performance art / 98-Festival / \n",
"0-Games / 23-Mobile phone / 29-Smartphone / 35-Minecraft / \n",
"319-Robot / \n",
"1-Vehicle / 4-Car / 1391-Kia / 1415-Kia Motors / \n",
"12-Food / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 178-Samsung Galaxy / 487-Samsung Electronics / 899-Samsung Galaxy S III / \n",
"23-Mobile phone / 29-Smartphone / 101-Telephone / \n",
"5-Dance / \n",
"31-Disc jockey / 559-Night / \n",
"2-Video game / 380-Assassin's Creed / 1048-Assassin's Creed / 1522-Assassin's Creed II / \n",
"20-Trailer / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"22-Nature / \n",
"10-Animal / 91-Fish / 258-Aquarium / \n",
"0-Games / 2-Video game / 37-Gadget / 55-Video game console / 292-Handheld game console / 847-Angry Birds / \n",
"2668-Duke Nukem / 3554-Duke Nukem Forever / \n",
"28-Fashion / \n",
"310-The Sims / 543-The Sims 2 / \n",
"3-Concert / 7-Musician / \n",
"122-Weight training / 204-Gym / 214-Muscle / 1389-Dumbbell / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 40-Road / 105-Motorcycling / \n",
"1-Vehicle / 22-Nature / 74-Truck / \n",
"0-Games / 2-Video game / 505-Mega Man / 1664-Zero / 3035-Mega Man Zero / \n",
"110-Album / \n",
"36-Piano / \n",
"1-Vehicle / 4-Car / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 172-Motocross / \n",
"1-Vehicle / 4-Car / 473-Ford Mustang / \n",
"122-Weight training / 204-Gym / 532-Barbell / \n",
"60-Basketball / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"0-Games / 326-Metin2 / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"8-Football / \n",
"0-Games / 15-Cartoon / 1650-The Idolmaster / 1755-The Idolmaster / \n",
"3-Concert / 5-Dance / 16-Performance art / 350-Circus / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / 472-Boeing 737 / 3433-Southwest Airlines / \n",
"3-Concert / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 101-Telephone / 554-iPhone 4 / \n",
"10-Animal / 174-Farm / 523-Cattle / 534-Milk / 4055-Milking / \n",
"205-Newscaster / \n",
"0-Games / 2-Video game / 43-Call of Duty / 302-Call of Duty 4: Modern Warfare / \n",
"12-Food / 26-Cooking / 32-Recipe / \n",
"1-Vehicle / 4-Car / 74-Truck / \n",
"31-Disc jockey / 39-Drums / \n",
"3-Concert / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 73-Ball / 76-Ball / 180-FIFA 15 / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 96-Soldier / 1543-Crysis / 2854-Stealth game / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 348-Fruit / 968-Pie / 1319-Apple / 3613-Apple pie / \n",
"3-Concert / 16-Performance art / \n",
"1-Vehicle / 4-Car / \n",
"3-Concert / 5-Dance / \n",
"5-Dance / 16-Performance art / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 194-Call of Duty: Modern Warfare 2 / 216-Call of Duty: Modern Warfare 3 / \n",
"18-Outdoor recreation / 83-Skateboarding / 221-Skatepark / 703-Roller skating / 1477-Aggressive inline skating / 2747-Inline skating / \n",
"2-Video game / 43-Call of Duty / 704-Call of Duty: World at War / \n",
"99-Christmas / \n",
"130-Gymnastics / \n",
"20-Trailer / 843-The Doctor / \n",
"0-Games / 2-Video game / 19-PC game / 55-Video game console / 320-Super Smash Bros. / 2165-Ganon / \n",
"1-Vehicle / 87-Boat / 444-Family / 753-Sailboat / \n",
"12-Food / \n",
"9-Music video / \n",
"0-Games / 8-Football / \n",
"75-Wrestling / 85-Combat / 117-Boxing / \n",
"3-Concert / 1854-Centella / \n",
"0-Games / 60-Basketball / 76-Ball / 168-Basketball moves / 623-Coach / 3291-Basketball Coach / \n",
"3-Concert / 21-Stadium / 46-Choir / 260-Prayer / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 403-Toyota / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"9-Music video / 28-Fashion / \n",
"0-Games / 2-Video game / 1638-Soulcalibur / 2877-Soulcalibur IV / \n",
"14-Guitar / 22-Nature / 156-Agriculture / 267-Mountain / \n",
"0-Games / 43-Call of Duty / 138-Call of Duty: Black Ops II / \n",
"10-Animal / 80-Horse / 384-Horse racing / 438-Mare / \n",
"10-Animal / 48-Pet / 71-Dog / \n",
"3-Concert / \n",
"9-Music video / \n",
"0-Games / 21-Stadium / 54-Highlight film / 81-Athlete / 562-Pitcher / \n",
"49-School / \n",
"9-Music video / \n",
"1-Vehicle / 11-Motorsport / 124-Tractor / 156-Agriculture / 249-Heavy equipment / 252-Tractor pulling / 363-Soil / 459-Plough / 618-Fiat Automobiles / \n",
"1-Vehicle / 4-Car / 25-Toy / \n",
"0-Games / 23-Mobile phone / 29-Smartphone / 1113-Toilet / \n",
"61-Art / 148-Painting / \n",
"1-Vehicle / 27-Motorcycle / 608-Wheelie / \n",
"93-Comedy / \n",
"0-Games / 2-Video game / 6-Animation / 241-Fighting game / 514-Tekken / 797-Tekken / 1470-Tekken Tag Tournament 2 / 4292-Heihachi Mishima / \n",
"1-Vehicle / 4-Car / 70-Driving / 118-Grand Theft Auto V / 297-Grand Theft Auto IV / 3074-Porsche Cayenne / \n",
"4501-The Adventures of Tintin / \n",
"9-Music video / \n",
"10-Animal / 80-Horse / 247-Stallion / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / 1303-Fullmetal Alchemist / \n",
"0-Games / 2-Video game / 8-Football / 90-Sports game / 180-FIFA 15 / 480-FIFA 13 / \n",
"1-Vehicle / 4-Car / 337-Advertising / 663-Street / \n",
"863-Body piercing / \n",
"0-Games / 2-Video game / 43-Call of Duty / 111-PlayStation 3 / 127-Xbox 360 / 194-Call of Duty: Modern Warfare 2 / 216-Call of Duty: Modern Warfare 3 / \n",
"6-Animation / \n",
"0-Games / 2-Video game / 72-iPhone / 189-iPad / 992-Rocket / \n",
"12-Food / 26-Cooking / 367-Drink / 904-Cocktail / 4295-Mojito / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 25-Toy / 140-LEGO / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / \n",
"14-Guitar / 36-Piano / \n",
"61-Art / 148-Painting / \n",
"10-Animal / 22-Nature / \n",
"0-Games / 1-Vehicle / 2-Video game / 19-PC game / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / 858-Carl Johnson / \n",
"31-Disc jockey / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 172-Motocross / 669-Extreme sport / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 64-Transport / 286-Bus / \n",
"96-Soldier / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 175-Airliner / \n",
"20-Trailer / \n",
"3-Concert / 804-Shopping mall / \n",
"2-Video game / 257-Counter-Strike / \n",
"0-Games / 78-Wedding / 271-Tank / 1167-Tanki Online / \n",
"0-Games / 6-Animation / 15-Cartoon / 3492-Gnome / \n",
"10-Animal / 80-Horse / \n",
"3-Concert / \n",
"28-Fashion / 3063-Friendship bracelet / \n",
"8-Football / \n",
"5-Dance / \n",
"238-Nail / 300-Nail art / 338-Manicure / \n",
"6-Animation / 2489-Sheriff Woody / \n",
"56-Hair / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / \n",
"337-Advertising / 776-Kindergarten / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 64-Transport / 124-Tractor / 252-Tractor pulling / \n",
"1-Vehicle / 4-Car / 349-Chevrolet / 406-Chevrolet / 761-Chevrolet Camaro / 864-Speedometer / 982-V8 engine / \n",
"1-Vehicle / 4-Car / 88-Machine / 114-Engine / 154-Wheel / \n",
"122-Weight training / 214-Muscle / \n",
"12-Food / \n",
"3-Concert / 7-Musician / 14-Guitar / 24-String instrument / \n",
"5-Dance / 2119-Splatter film / \n",
"7-Musician / 14-Guitar / 63-Acoustic guitar / 488-Door / \n",
"22-Nature / 28-Fashion / 206-Shoe / 333-Nike; Inc. / 368-Sneakers / 985-Strawberry / 1423-Nike Air Max / \n",
"3-Concert / \n",
"25-Toy / 42-Fishing / 140-LEGO / \n",
"5-Dance / 13-Musical ensemble / 98-Festival / 203-Marching band / 250-Parade / \n",
"0-Games / 1-Vehicle / 4-Car / 6-Animation / 15-Cartoon / 25-Toy / 74-Truck / \n",
"31-Disc jockey / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 1056-Porsche 911 / \n",
"67-Cycling / 69-Bicycle / 154-Wheel / 519-Recreational vehicle / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / \n",
"1-Vehicle / 4-Car / 74-Truck / \n",
"8-Football / \n",
"0-Games / 8-Football / 21-Stadium / 81-Athlete / \n",
"6-Animation / 15-Cartoon / 111-PlayStation 3 / 4286-Atelier / \n",
"9-Music video / \n",
"0-Games / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"133-Nightclub / \n",
"0-Games / 1852-Final Fantasy XI / \n",
"1-Vehicle / 4-Car / 23-Mobile phone / 29-Smartphone / \n",
"3-Concert / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 210-Cookware and bakeware / 597-Soup / 1381-Garlic / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / \n",
"1-Vehicle / 20-Trailer / 50-Aircraft / 766-President of the United States / \n",
"9-Music video / 15-Cartoon / 524-Littlest Pet Shop / \n",
"1484-Final Fantasy VIII / \n",
"260-Prayer / \n",
"342-Windows Media Video / 366-Dragon / \n",
"1-Vehicle / 1699-Cart / 3573-Golf cart / \n",
"12-Food / 26-Cooking / 52-Dish / 58-Cuisine / 215-Eating / 301-Meal / 389-Restaurant / \n",
"6-Animation / 15-Cartoon / 1724-Homer Simpson / \n",
"22-Nature / 46-Choir / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"0-Games / 19-PC game / 51-Strategy video game / 373-Clash of Clans / \n",
"12-Food / 3250-Chubby Bunny / \n",
"1669-Vodka / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 73-Ball / 76-Ball / 81-Athlete / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 247-Stallion / 289-Dressage / 438-Mare / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 331-iPod touch / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / 265-Electronic keyboard / 592-Synthesia / \n",
"1-Vehicle / 4-Car / 1243-Mirror / \n",
"125-Television / \n",
"22-Nature / 59-Winter sport / 177-Skiing / 267-Mountain / 347-Climbing / \n",
"110-Album / \n",
"2155-Unicorn / \n",
"1137-Crash Bandicoot / 4629-Crash Twinsanity / \n",
"6-Animation / 15-Cartoon / 877-Hatsune Miku: Project DIVA / 1648-Hatsune Miku: Project DIVA F / \n",
"5-Dance / 78-Wedding / 190-Bride / 889-Swing / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / \n",
"0-Games / 3512-Just Dance 2015 / \n",
"9-Music video / 31-Disc jockey / \n",
"3-Concert / 7-Musician / 16-Performance art / 30-Drummer / \n",
"3-Concert / 7-Musician / \n",
"9-Music video / \n",
"9-Music video / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / \n",
"0-Games / 2-Video game / 19-PC game / 734-DayZ / \n",
"61-Art / 148-Painting / 303-Paint / 873-Acrylic paint / 936-Watercolor paint / \n",
"93-Comedy / 355-Comedian / \n",
"273-News program / 913-President / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 327-Mud / 853-Dune buggy / \n",
"28-Fashion / 327-Mud / 757-Boot / 785-Jeans / 3505-Wellington boot / \n",
"39-Drums / 724-Rock Band / \n",
"1-Vehicle / 87-Boat / 1643-Submarine / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 40-Road / \n",
"1-Vehicle / 4-Car / 1511-Toyota Land Cruiser / \n",
"12-Food / \n",
"6-Animation / 15-Cartoon / 2176-Fist of the North Star / 3349-Kenshiro / \n",
"3-Concert / 7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"12-Food / 26-Cooking / 389-Restaurant / \n",
"6-Animation / 701-Beyblade / \n",
"10-Animal / 48-Pet / 71-Dog / 488-Door / \n",
"1229-Magnet / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"237-Final Fantasy / \n",
"3-Concert / 16-Performance art / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 422-Call of Duty: Ghosts / \n",
"10-Animal / 48-Pet / 71-Dog / \n",
"130-Gymnastics / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 21-Stadium / 57-Race track / 77-Arena / \n",
"68-Lighting / 263-Light / 2703-Light fixture / \n",
"6-Animation / 23-Mobile phone / 29-Smartphone / \n",
"1-Vehicle / 4-Car / 67-Cycling / 69-Bicycle / 154-Wheel / 520-Bicycle frame / \n",
"2224-Sherlock Holmes / \n",
"49-School / \n",
"10-Animal / 71-Dog / \n",
"5-Dance / 16-Performance art / 95-Talent show / \n",
"8-Football / 205-Newscaster / \n",
"5-Dance / 115-Ballet / \n",
"248-Hotel / \n",
"1975-Amiga / \n",
"3-Concert / 16-Performance art / \n",
"15-Cartoon / 99-Christmas / 2047-Popeye / \n",
"1-Vehicle / 50-Aircraft / 121-Photography / 1955-Letter / \n",
"6-Animation / 15-Cartoon / 183-Manga / 396-One Piece / \n",
"93-Comedy / \n",
"1-Vehicle / 4-Car / 201-Grand Theft Auto: San Andreas / \n",
"42-Fishing / 293-Slide show / \n",
"5-Dance / 16-Performance art / \n",
"20-Trailer / 126-The Walt Disney Company / \n",
"3-Concert / 7-Musician / 16-Performance art / 30-Drummer / \n",
"130-Gymnastics / \n",
"0-Games / 2-Video game / 320-Super Smash Bros. / 584-Super Mario Bros. / 685-Luigi / 887-King Koopa / 3072-Super Smash Bros. / \n",
"0-Games / 54-Highlight film / 256-Tennis / \n",
"3-Concert / \n",
"45-Cosmetics / 2184-Lush / \n",
"10-Animal / 22-Nature / 132-Bird / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 1072-Total War / 2674-Rome: Total War / \n",
"12-Food / 28-Fashion / 45-Cosmetics / 282-Skin / 385-Face / 784-Honey / 3404-Oatmeal / \n",
"6-Animation / 15-Cartoon / 4714-Air Gear / \n",
"1-Vehicle / 33-Weapon / 271-Tank / \n",
"14-Guitar / 36-Piano / 53-Keyboard / 107-Musical keyboard / 150-Violin / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / \n",
"2-Video game / 320-Super Smash Bros. / \n",
"0-Games / 1-Vehicle / 2-Video game / 35-Minecraft / 87-Boat / 308-Ship / \n",
"0-Games / 88-Machine / 240-Arcade game / 3591-Pump It Up Fiesta 2 / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 43-Call of Duty / 138-Call of Duty: Black Ops II / \n",
"1-Vehicle / 42-Fishing / 87-Boat / 2755-Lithium polymer battery / \n",
"3-Concert / 68-Lighting / \n",
"1-Vehicle / 4-Car / 449-Honda / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"3-Concert / \n",
"6-Animation / 15-Cartoon / 61-Art / 1164-Cartoon Network / \n",
"0-Games / 10-Animal / 17-Racing / 80-Horse / 384-Horse racing / \n",
"0-Games / 2-Video game / 6-Animation / 89-Comics / 396-One Piece / 1312-Dynasty Warriors / 4203-One Piece: Pirate Warriors / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 406-Chevrolet / 952-Chevrolet Corvette / \n",
"6-Animation / 97-Drawing / 183-Manga / 408-Sketch / 1674-Ouran High School Host Club / \n",
"28-Fashion / 45-Cosmetics / 56-Hair / 106-Hairstyle / 193-Eye liner / \n",
"47-Personal computer / 145-Tablet computer / 189-iPad / 525-Printer / 2095-Cash / 2913-Drawer / \n",
"93-Comedy / 355-Comedian / \n",
"1-Vehicle / 4-Car / 245-BMW / 270-Sedan / 1233-BMW 3 Series / 2130-Car dealership / \n",
"45-Cosmetics / 56-Hair / 291-Rouge / \n",
"1507-Calligraphy / 4460-Chinese calligraphy / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 236-Baking / 442-Bread / 485-Dough / 600-Flour / 662-Pizza / 2728-Yeast / \n",
"1-Vehicle / 18-Outdoor recreation / 42-Fishing / 548-Parachuting / 669-Extreme sport / 765-Parachute / \n",
"0-Games / 2-Video game / 35-Minecraft / 84-Snow / \n",
"5-Dance / 16-Performance art / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / 318-Church / 397-Organ / \n",
"694-Tai chi / \n",
"9-Music video / \n",
"2019-Dental braces / \n",
"3-Concert / 16-Performance art / 68-Lighting / \n",
"0-Games / 2-Video game / 90-Sports game / 180-FIFA 15 / \n",
"1-Vehicle / 4-Car / 70-Driving / 3439-Compact car / \n",
"28-Fashion / 206-Shoe / 333-Nike; Inc. / 368-Sneakers / \n",
"9-Music video / \n",
"5-Dance / 16-Performance art / \n",
"639-Fiddle / \n",
"60-Basketball / \n",
"25-Toy / 435-Action figure / \n",
"7-Musician / 230-Brass instrument / 1574-French horn / \n",
"0-Games / 489-Board game / 1966-The Lord of the Rings / \n",
"0-Games / 756-Tibia / \n",
"201-Grand Theft Auto: San Andreas / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"10-Animal / 48-Pet / 71-Dog / \n",
"1-Vehicle / 18-Outdoor recreation / 87-Boat / 114-Engine / 147-River / 225-Lake / 498-Motorboat / \n",
"0-Games / 60-Basketball / 77-Arena / 168-Basketball moves / 255-Slam dunk / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 135-Drifting / \n",
"1-Vehicle / 4-Car / \n",
"1-Vehicle / 4-Car / 13-Musical ensemble / 27-Motorcycle / 1924-Symbol / \n",
"12-Food / 587-Clay / 695-Balloon / \n",
"3-Concert / 7-Musician / 16-Performance art / 30-Drummer / \n",
"155-Camera / 1348-Panasonic Corporation / \n",
"6-Animation / 15-Cartoon / 123-Naruto / 310-The Sims / 543-The Sims 2 / 622-Sakura Haruno / \n",
"3-Concert / 5-Dance / 7-Musician / \n",
"1-Vehicle / 4-Car / 1245-Alarm device / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 434-Ninja / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"47-Personal computer / 145-Tablet computer / \n",
"61-Art / 2905-Warhammer Online: Age of Reckoning / \n",
"0-Games / 256-Tennis / \n",
"0-Games / 59-Winter sport / 81-Athlete / 112-Ice skating / 142-Hockey / 312-Ice rink / \n",
"0-Games / 6-Animation / \n",
"337-Advertising / \n",
"7-Musician / 13-Musical ensemble / 21-Stadium / 49-School / 79-American football / 203-Marching band / 306-High school / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 39-Drums / 44-Drums / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 161-Train station / \n",
"2830-Super Mario Sunshine / \n",
"1-Vehicle / 4-Car / 234-Ford / 344-Coupé / 406-Chevrolet / 509-Classic car / 1148-Hot rod / \n",
"12-Food / 215-Eating / 1701-French fries / \n",
"6-Animation / 10-Animal / 15-Cartoon / 132-Bird / 718-Duck / 1445-Donald Duck / \n",
"1-Vehicle / 50-Aircraft / 499-Cloud / \n",
"0-Games / 2-Video game / 19-PC game / 1630-Warframe / \n",
"136-Gardening / 2928-Cereal / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 94-Dashcam / 169-Rallying / \n",
"22-Nature / 86-Plant / 136-Gardening / \n",
"5-Dance / 98-Festival / 250-Parade / 253-Carnival / \n",
"12-Food / 88-Machine / 539-Home appliance / 818-Washing machine / 827-Pump / 1456-Clothes dryer / \n",
"0-Games / 2-Video game / 211-RuneScape / \n",
"8-Football / 54-Highlight film / 65-Kick / \n",
"3-Concert / 16-Performance art / 38-Orchestra / \n",
"93-Comedy / \n",
"1-Vehicle / 4-Car / 1547-Passenger / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"61-Art / 97-Drawing / 242-Paper / 2162-Marker pen / \n",
"122-Weight training / 204-Gym / 481-Kayak / \n",
"0-Games / 5-Dance / \n",
"122-Weight training / 214-Muscle / 1250-Biceps / \n",
"22-Nature / 86-Plant / 136-Gardening / 137-Tree / 1460-Herb / 1969-Shrub / 4644-Vine / \n",
"1-Vehicle / 162-Medicine / \n",
"3-Concert / 7-Musician / 24-String instrument / \n",
"1-Vehicle / 4-Car / 40-Road / 64-Transport / 74-Truck / 482-Asphalt / \n",
"3-Concert / 5-Dance / 13-Musical ensemble / 98-Festival / 1289-Caporales / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 989-Warface / \n",
"0-Games / 2-Video game / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / \n",
"1-Vehicle / 120-Winter / 262-Ice / 308-Ship / \n",
"14-Guitar / \n",
"567-Compact disc / \n",
"9-Music video / \n",
"624-Number / 1507-Calligraphy / \n",
"16-Performance art / 78-Wedding / \n",
"0-Games / 6-Animation / 319-Robot / \n",
"9-Music video / 110-Album / \n",
"15-Cartoon / 61-Art / 97-Drawing / 237-Final Fantasy / 408-Sketch / 538-Illustration / \n",
"120-Winter / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 169-Rallying / \n",
"1-Vehicle / 27-Motorcycle / \n",
"0-Games / 35-Minecraft / 246-Furniture / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 185-Dragon Ball / 244-Goku / 418-Gohan / 869-Dragon Ball Z: Budokai Tenkaichi / \n",
"14-Guitar / 162-Medicine / \n",
"0-Games / 774-Galaxy / 1397-Super Mario Galaxy / 1759-Super Mario Galaxy 2 / \n",
"1-Vehicle / 4-Car / 25-Toy / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 40-Road / 105-Motorcycling / 207-Exhaust system / \n",
"6-Animation / 15-Cartoon / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 197-Trail / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 134-Call of Duty: Black Ops / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"8-Football / \n",
"7-Musician / 14-Guitar / 63-Acoustic guitar / 2039-Soybean / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / 912-Silver / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 57-Race track / 169-Rallying / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"14-Guitar / \n",
"242-Paper / 745-Writing / 882-Pen / 2454-Notebook / \n",
"3-Concert / 133-Nightclub / \n",
"0-Games / 2-Video game / 19-PC game / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / \n",
"4676-Elphaba / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 70-Driving / 94-Dashcam / 169-Rallying / \n",
"3-Concert / \n",
"38-Orchestra / \n",
"0-Games / 2-Video game / 55-Video game console / 3567-SOCOM / \n",
"31-Disc jockey / \n",
"3-Concert / 133-Nightclub / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 86-Plant / 109-Cooking show / 184-Vegetable / 210-Cookware and bakeware / 597-Soup / \n",
"28-Fashion / \n",
"6-Animation / 15-Cartoon / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 556-Skateboarding trick / 1315-Kickflip / \n",
"5-Dance / 7-Musician / 16-Performance art / 38-Orchestra / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"467-Chipmunk / \n",
"12-Food / 26-Cooking / 32-Recipe / 232-Cake / 236-Baking / 295-Chocolate / 442-Bread / 485-Dough / 560-Sugar / 578-Cookie / 600-Flour / 759-Icing / 2699-Chocolate chip / 3211-Chocolate chip cookie / \n",
"5-Dance / 98-Festival / 988-Ibiza / \n",
"8-Football / \n",
"59-Winter sport / 84-Snow / 281-Snowboarding / 501-Snowboard / \n",
"1-Vehicle / 27-Motorcycle / \n",
"2-Video game / 229-Halo / 470-Halo 3 / \n",
"1-Vehicle / 4-Car / 28-Fashion / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 274-Meat / 454-Sauce / 752-Pasta / 1230-Italian food / 1517-Spaghetti / 2562-Meatball / \n",
"6-Animation / 522-Helmet / \n",
"1-Vehicle / 11-Motorsport / 172-Motocross / \n",
"0-Games / 6-Animation / 25-Toy / \n",
"1-Vehicle / 2-Video game / 6-Animation / \n",
"0-Games / 19-PC game / 185-Dragon Ball / 244-Goku / 837-Roblox / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 101-Telephone / 178-Samsung Galaxy / 802-Google Nexus / 2929-Galaxy Nexus / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / \n",
"10-Animal / 48-Pet / 132-Bird / 1799-Domestic canary / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 6-Animation / \n",
"12-Food / 26-Cooking / 32-Recipe / 475-Cheese / 534-Milk / 786-Juice / 1081-Lemon / 1193-Orange / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 118-Grand Theft Auto V / 1393-Lightsaber / \n",
"46-Choir / \n",
"1-Vehicle / 4-Car / 70-Driving / 299-Volkswagen Passenger Cars / 526-Volkswagen / 1761-Volkswagen Jetta / \n",
"9-Music video / 25-Toy / 524-Littlest Pet Shop / \n",
"1-Vehicle / 11-Motorsport / 3311-Hovercraft / \n",
"14-Guitar / 1346-Wood carving / \n",
"0-Games / 2-Video game / 1981-Commodore 64 / \n",
"6-Animation / 15-Cartoon / 89-Comics / 123-Naruto / 183-Manga / 1556-Dōjinshi / \n",
"3-Concert / 7-Musician / \n",
"2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / 47-Personal computer / 72-iPhone / 101-Telephone / 145-Tablet computer / 178-Samsung Galaxy / 1629-Samsung Galaxy Note / 1776-Samsung Galaxy Note II / \n",
"0-Games / 2-Video game / 19-PC game / 159-World of Warcraft / 171-Warcraft / 3615-Monk / \n",
"2-Video game / 37-Gadget / 55-Video game console / 292-Handheld game console / 335-Nintendo 3DS / \n",
"2-Video game / 82-Airplane / 194-Call of Duty: Modern Warfare 2 / 216-Call of Duty: Modern Warfare 3 / 1673-Airport terminal / \n",
"7-Musician / 14-Guitar / 63-Acoustic guitar / \n",
"0-Games / 2-Video game / 2628-Sly Cooper / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 135-Drifting / \n",
"31-Disc jockey / \n",
"7-Musician / 46-Choir / 1338-Devil / \n",
"1-Vehicle / 4-Car / 70-Driving / 1285-Parking / 1314-Garage / \n",
"9-Music video / 31-Disc jockey / 133-Nightclub / \n",
"1088-Infomercial / 3301-Mop / \n",
"287-Human swimming / \n",
"2-Video game / 387-Spider-Man / \n",
"2-Video game / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / 1299-Shadow the Hedgehog / 3321-Shadow the Hedgehog / \n",
"350-Circus / 788-Juggling / \n",
"5-Dance / 16-Performance art / 22-Nature / \n",
"356-Star / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 617-Maize / \n",
"0-Games / 2-Video game / 910-Dragon Quest / 4411-Dragon Quest V: Hand of the Heavenly Bride / \n",
"1-Vehicle / 27-Motorcycle / 342-Windows Media Video / 608-Wheelie / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"110-Album / 242-Paper / \n",
"6-Animation / 25-Toy / 200-Doll / 412-Monster / 503-Monster High / \n",
"1-Vehicle / 4-Car / 74-Truck / 1070-Semi-trailer truck / \n",
"8-Football / 76-Ball / 2611-Wiffle ball / \n",
"9-Music video / \n",
"466-Gift / 2507-Macramé / \n",
"22-Nature / 84-Snow / 225-Lake / 267-Mountain / \n",
"28-Fashion / \n",
"59-Winter sport / 112-Ice skating / 115-Ballet / 316-Figure skating / \n",
"0-Games / 165-League of Legends / 296-Saxophone / \n",
"0-Games / 8-Football / 73-Ball / \n",
"42-Fishing / 222-Surfing / \n",
"278-GoPro / \n",
"476-Textile / 1403-Flower bouquet / 2329-Felt / \n",
"61-Art / 479-Graffiti / 1846-Mural / \n",
"452-Watch / 2168-G-Shock / \n",
"10-Animal / 80-Horse / 247-Stallion / \n",
"0-Games / 2-Video game / 1777-Ratchet / 1824-Clank / 1859-Ratchet & Clank / 4423-Ratchet & Clank / \n",
"0-Games / 8-Football / 20-Trailer / 79-American football / 90-Sports game / 561-Madden NFL / 1813-Madden NFL 13 / 2916-Madden NFL 12 / \n",
"1-Vehicle / 4-Car / 6-Animation / 15-Cartoon / 74-Truck / 88-Machine / 124-Tractor / 433-Construction / \n",
"163-Running / \n",
"0-Games / 21-Stadium / 54-Highlight film / 163-Running / \n",
"18-Outdoor recreation / 217-Hunting / 706-Deer / \n",
"5-Dance / 430-Ballroom dance / 678-Latin dance / \n",
"0-Games / 6-Animation / 15-Cartoon / \n",
"10-Animal / 48-Pet / 71-Dog / 824-Terrier / 1252-Dog agility / \n",
"0-Games / 2-Video game / 55-Video game console / 292-Handheld game console / 398-The Legend of Zelda / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"113-House / 170-Home improvement / 879-Ceiling / \n",
"3-Concert / 31-Disc jockey / \n",
"60-Basketball / 255-Slam dunk / \n",
"5-Dance / 9-Music video / \n",
"6-Animation / 15-Cartoon / 1771-The Pink Panther / \n",
"66-Bollywood / \n",
"122-Weight training / 204-Gym / 1584-Kettlebell / \n",
"1-Vehicle / 10-Animal / 18-Outdoor recreation / 42-Fishing / 87-Boat / 91-Fish / 268-Recreational fishing / 698-Fisherman / \n",
"925-Brain / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 30-Drummer / \n",
"222-Surfing / 672-Bodyboarding / 815-Surfboard / \n",
"102-Building / 242-Paper / 339-Architecture / 342-Windows Media Video / 1214-Skyscraper / \n",
"468-Biology / \n",
"1-Vehicle / 810-Village / 4324-Amish / \n",
"3-Concert / \n",
"9-Music video / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / \n",
"17-Racing / 67-Cycling / 69-Bicycle / 570-Road bicycle racing / \n",
"3-Concert / 16-Performance art / \n",
"8-Football / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 39-Drums / \n",
"110-Album / \n",
"12-Food / 215-Eating / 1481-Sunglasses / \n",
"0-Games / 241-Fighting game / 764-The King of Fighters / 2354-The King of Fighters 2002 / \n",
"114-Engine / 527-Battery / 1089-Wire / \n",
"23-Mobile phone / 37-Gadget / 47-Personal computer / 145-Tablet computer / 189-iPad / \n",
"10-Animal / 48-Pet / 71-Dog / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 74-Truck / \n",
"1-Vehicle / 50-Aircraft / 712-Outer space / 1992-Space Shuttle / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 1134-Taxicab / \n",
"61-Art / 148-Painting / \n",
"0-Games / 2-Video game / 445-Guitar Hero / 965-Guitar Hero III: Legends of Rock / \n",
"242-Paper / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 25-Toy / 92-Radio-controlled model / 202-Radio-controlled car / 482-Asphalt / \n",
"0-Games / 489-Board game / 799-Chess / 922-Tabletop game / \n",
"0-Games / 1000-AdventureQuest Worlds / \n",
"9-Music video / \n",
"5-Dance / 16-Performance art / \n",
"12-Food / 26-Cooking / 32-Recipe / 226-Dessert / 232-Cake / 236-Baking / 759-Icing / 770-Banana / 795-Cupcake / \n",
"3-Concert / 7-Musician / 342-Windows Media Video / 376-Flute / 726-Clarinet / \n",
"45-Cosmetics / 235-Lipstick / 238-Nail / 300-Nail art / 324-Finger / 338-Manicure / 2235-Avon Products / \n",
"5-Dance / \n",
"36-Piano / 137-Tree / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"359-Harry Potter (Literary Series) / \n",
"2-Video game / 1199-Kinect / \n",
"124-Tractor / \n",
"209-University / \n",
"0-Games / 2-Video game / 6-Animation / 257-Counter-Strike / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 42-Fishing / 225-Lake / 275-Diving / 288-Underwater / 381-Underwater diving / \n",
"10-Animal / 86-Plant / 91-Fish / 258-Aquarium / 1659-Walmart / \n",
"89-Comics / \n",
"1-Vehicle / 4-Car / 70-Driving / 600-Flour / 630-Superman / \n",
"3-Concert / 16-Performance art / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / 3987-Tight end / \n",
"164-Beach / 222-Surfing / 278-GoPro / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"5-Dance / 16-Performance art / 95-Talent show / 980-Solo dance / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"1-Vehicle / 10-Animal / 18-Outdoor recreation / 42-Fishing / 87-Boat / 91-Fish / 698-Fisherman / \n",
"3-Concert / 5-Dance / 46-Choir / 49-School / \n",
"6-Animation / \n",
"3-Concert / \n",
"67-Cycling / 69-Bicycle / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 231-Supercar / 711-Gran Turismo / 953-Gran Turismo 5 / 1763-Lotus Cars / \n",
"0-Games / 2-Video game / 34-Action-adventure game / \n",
"0-Games / 35-Minecraft / \n",
"423-Coast / 778-Gliding / \n",
"45-Cosmetics / 235-Lipstick / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 19-PC game / 51-Strategy video game / 165-League of Legends / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 81-Athlete / \n",
"18-Outdoor recreation / 28-Fashion / 894-Jacket / \n",
"2791-Transformer / \n",
"13-Musical ensemble / \n",
"81-Athlete / 562-Pitcher / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / 38-Orchestra / \n",
"38-Orchestra / 541-Black-and-white / \n",
"5-Dance / \n",
"0-Games / 25-Toy / 126-The Walt Disney Company / 417-Play-Doh / 679-Mickey Mouse / 1234-Mouse / 1327-Minnie Mouse / 2161-Playground / \n",
"96-Soldier / 257-Counter-Strike / \n",
"9-Music video / 1058-Volcano / \n",
"12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / 109-Cooking show / 485-Dough / 662-Pizza / \n",
"6-Animation / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / \n",
"1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 111-PlayStation 3 / 1763-Lotus Cars / 3893-F1 2013 / \n",
"78-Wedding / \n",
"9-Music video / \n",
"147-River / 481-Kayak / 1305-Canoeing / 3858-Canoe Slalom / \n",
"9-Music video / \n",
"12-Food / 26-Cooking / 58-Cuisine / \n",
"0-Games / 756-Tibia / \n",
"59-Winter sport / 84-Snow / 177-Skiing / 305-Ski / 789-Alpine skiing / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / \n",
"6-Animation / 89-Comics / \n",
"7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / 44-Drums / 49-School / 146-Snare drum / 203-Marching band / \n",
"64-Transport / \n",
"0-Games / 2-Video game / 2633-Soulcalibur V / \n",
"0-Games / 2-Video game / 961-Sega Genesis / \n",
"6-Animation / 15-Cartoon / 200-Doll / 412-Monster / 503-Monster High / \n",
"2-Video game / 6-Animation / 477-Transformers / 990-Optimus Prime / 1900-Autobot / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 352-Need for Speed / 2407-Need for Speed: The Run / \n",
"78-Wedding / 190-Bride / \n",
"0-Games / 19-PC game / 35-Minecraft / 229-Halo / \n",
"116-Pokémon / 372-Playing card / 1021-Booster pack / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"155-Camera / 222-Surfing / 278-GoPro / \n",
"5-Dance / 13-Musical ensemble / 49-School / 79-American football / 203-Marching band / \n",
"0-Games / 8-Football / 54-Highlight film / 76-Ball / \n",
"0-Games / 8-Football / \n",
"322-Mixtape / \n",
"259-Fire / 371-Firefighter / \n",
"12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / 109-Cooking show / 226-Dessert / 557-Salad / 671-Tomato / 1769-Olive / \n",
"5-Dance / 16-Performance art / \n",
"0-Games / 1938-Rust / \n",
"31-Disc jockey / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"0-Games / 8-Football / 65-Kick / 73-Ball / 76-Ball / \n",
"28-Fashion / 45-Cosmetics / 56-Hair / 106-Hairstyle / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 2-Video game / 1091-Saints Row / 2495-Saints Row 2 / \n",
"10-Animal / 48-Pet / 71-Dog / \n",
"3-Concert / 5-Dance / 31-Disc jockey / 133-Nightclub / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 420-The Elder Scrolls V: Skyrim / 441-The Elder Scrolls / \n",
"12-Food / 179-Kitchen / 539-Home appliance / 1439-Refrigerator / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 74-Truck / 158-Tire / 167-Four-wheel drive / 198-Off-road vehicle / 327-Mud / 375-Pickup truck / 515-Dodge / 565-Mud bogging / 854-Diesel engine / 1437-Ram Trucks / 1651-Dodge Ram / \n",
"136-Gardening / \n",
"278-GoPro / 283-Rock / 1416-Cliff / \n",
"10-Animal / 80-Horse / 247-Stallion / \n",
"780-Fat / \n",
"1-Vehicle / 42-Fishing / 87-Boat / 517-Audi / 579-Yacht / 1309-Sailing ship / \n",
"3-Concert / 56-Hair / 106-Hairstyle / 323-Toddler / 1113-Toilet / \n",
"5-Dance / 4667-Gold's Gym / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 135-Drifting / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"148-Painting / 170-Home improvement / 303-Paint / 1350-Plaster / \n",
"56-Hair / 106-Hairstyle / \n",
"99-Christmas / \n",
"68-Lighting / 263-Light / 1375-Laser lighting display / \n",
"1-Vehicle / 4-Car / 124-Tractor / 249-Heavy equipment / \n",
"12-Food / 26-Cooking / 32-Recipe / 600-Flour / \n",
"85-Combat / \n",
"0-Games / 2-Video game / 426-Resident Evil / 2903-Umbrella / \n",
"271-Tank / 409-World of Tanks / \n",
"0-Games / 59-Winter sport / 81-Athlete / 112-Ice skating / 142-Hockey / 312-Ice rink / 1973-Hockey puck / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / 133-Nightclub / \n",
"6-Animation / 15-Cartoon / 61-Art / 97-Drawing / \n",
"162-Medicine / 890-Brush / 3879-Tooth brushing / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"24-String instrument / 465-Microphone / \n",
"0-Games / 2-Video game / 596-Monster Hunter / 1339-Monster Hunter Freedom Unite / \n",
"2-Video game / 6-Animation / 1150-iPad 2 / \n",
"1721-Tetris / \n",
"3296-Transistor / \n",
"516-Headphones / \n",
"253-Carnival / \n",
"1638-Soulcalibur / 4628-Soulcalibur III / \n",
"3-Concert / 7-Musician / 14-Guitar / 63-Acoustic guitar / 356-Star / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 218-Eyelash / 235-Lipstick / 291-Rouge / \n",
"3-Concert / 5-Dance / 16-Performance art / 78-Wedding / \n",
"330-Money / 2390-Dollar / 2722-United States Dollar / 2937-Banknote / \n",
"8-Football / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 178-Samsung Galaxy / 1727-Samsung Galaxy Note 3 / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 2318-Screenshot / \n",
"2-Video game / 20-Trailer / 1196-Dragon Age / 2700-Dragon Age II / \n",
"0-Games / 75-Wrestling / \n",
"12-Food / \n",
"1-Vehicle / 4-Car / 345-Loudspeaker / 484-Subwoofer / \n",
"0-Games / \n",
"3-Concert / 9-Music video / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / 81-Athlete / \n",
"18-Outdoor recreation / 49-School / 83-Skateboarding / 108-Skateboard / 482-Asphalt / \n",
"5-Dance / \n",
"5-Dance / 16-Performance art / \n",
"1-Vehicle / 4-Car / 20-Trailer / \n",
"10-Animal / 42-Fishing / 258-Aquarium / 288-Underwater / \n",
"3-Concert / \n",
"12-Food / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / \n",
"3-Concert / 7-Musician / \n",
"5-Dance / 13-Musical ensemble / \n",
"315-Knife / \n",
"10-Animal / 642-Leaf / \n",
"10-Animal / 22-Nature / 42-Fishing / 91-Fish / 275-Diving / 601-Shark / \n",
"10-Animal / 18-Outdoor recreation / 42-Fishing / 91-Fish / 268-Recreational fishing / \n",
"0-Games / 2-Video game / 6-Animation / 1777-Ratchet / 1824-Clank / 1859-Ratchet & Clank / 4423-Ratchet & Clank / \n",
"783-Zee TV / \n",
"5-Dance / 16-Performance art / 115-Ballet / 980-Solo dance / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 135-Drifting / \n",
"6-Animation / 15-Cartoon / \n",
"259-Fire / 371-Firefighter / \n",
"8-Football / \n",
"0-Games / 42-Fishing / 1945-Medal game / \n",
"5-Dance / \n",
"1-Vehicle / 4-Car / 207-Exhaust system / 618-Fiat Automobiles / \n",
"21-Stadium / 54-Highlight film / 81-Athlete / 163-Running / 1405-Primary school / \n",
"3-Concert / 5-Dance / \n",
"125-Television / \n",
"3-Concert / 68-Lighting / \n",
"45-Cosmetics / 173-Eye shadow / 325-Vampire / 2556-Halloween costume / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 380-Assassin's Creed / 1140-Assassin's Creed III / \n",
"148-Painting / 744-Window / 1794-Curtain / \n",
"6-Animation / \n",
"93-Comedy / \n",
"10-Animal / 48-Pet / 71-Dog / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / \n",
"1-Vehicle / 4-Car / 70-Driving / 167-Four-wheel drive / 198-Off-road vehicle / 283-Rock / 360-Jeep / \n",
"31-Disc jockey / \n",
"0-Games / 1681-Mu Online / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 398-The Legend of Zelda / 692-Link / 3846-The Legend of Zelda: A Link to the Past / \n",
"5-Dance / \n",
"1-Vehicle / \n",
"89-Comics / 269-Comic book / \n",
"31-Disc jockey / \n",
"78-Wedding / \n",
"208-Wood / 981-Sculpture / 1073-Chainsaw / 1346-Wood carving / \n",
"3-Concert / 68-Lighting / 77-Arena / \n",
"209-University / \n",
"9-Music video / \n",
"0-Games / 116-Pokémon / 186-Pokémon / \n",
"28-Fashion / 45-Cosmetics / 785-Jeans / 894-Jacket / 1822-Denim / \n",
"22-Nature / 147-River / 825-Stream / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / \n",
"0-Games / 8-Football / 54-Highlight film / 76-Ball / \n",
"3467-Gingerbread / \n",
"1-Vehicle / \n",
"89-Comics / 269-Comic book / 294-Batman / 855-Hulk / 897-Avengers / 2452-Thor / 3412-Daredevil / \n",
"9-Music video / \n",
"0-Games / 8-Football / 21-Stadium / 76-Ball / 77-Arena / 90-Sports game / 180-FIFA 15 / 453-Pro Evolution Soccer / 1719-Pro Evolution Soccer 2015 / \n",
"47-Personal computer / 131-Computer / 228-Laptop / \n",
"574-Elevator / 1285-Parking / 1746-Otis Elevator Company / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / \n",
"3-Concert / 5-Dance / 16-Performance art / 49-School / 95-Talent show / \n",
"3-Concert / 5-Dance / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 277-Sport utility vehicle / 517-Audi / 4026-Audi Q7 / \n",
"97-Drawing / 148-Painting / 769-Red Bull / 2992-Energy drink / 3139-Beverage can / \n",
"1-Vehicle / 2-Video game / 2401-Wolfenstein / 4432-Wolfenstein: The New Order / \n",
"9-Music video / \n",
"85-Combat / 1146-Wing Chun / \n",
"1-Vehicle / 11-Motorsport / 124-Tractor / 156-Agriculture / 174-Farm / 252-Tractor pulling / 931-Combine Harvester / \n",
"113-House / 170-Home improvement / 276-Room / 416-Apartment / 580-Bedroom / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 81-Athlete / \n",
"2062-Kidney / \n",
"0-Games / 2-Video game / 43-Call of Duty / 216-Call of Duty: Modern Warfare 3 / \n",
"85-Combat / 117-Boxing / 681-Leather / 1016-Glove / \n",
"573-Paragliding / 765-Parachute / \n",
"1357-Ceramic / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 378-Drag racing / 449-Honda / 482-Asphalt / 552-Honda Civic / \n",
"25-Toy / 2689-Furby / \n",
"0-Games / 1-Vehicle / 19-PC game / 33-Weapon / 271-Tank / 409-World of Tanks / \n",
"5-Dance / \n",
"12-Food / 367-Drink / 560-Sugar / 1044-Coca-Cola / 1224-Cola / 1589-Soft drink / 2032-Pepsi / \n",
"1-Vehicle / 27-Motorcycle / \n",
"0-Games / 2-Video game / 4054-The Forest / \n",
"5-Dance / 678-Latin dance / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / 1340-Airbus A380 / 3671-Instrument landing system / \n",
"3809-Marriott International / \n",
"134-Call of Duty: Black Ops / \n",
"3-Concert / 21-Stadium / \n",
"1-Vehicle / \n",
"18-Outdoor recreation / 163-Running / 267-Mountain / 283-Rock / 347-Climbing / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 105-Motorcycling / 434-Ninja / 1380-Kawasaki motorcycles / 2345-Kawasaki Ninja 250R / 2404-Kawasaki Ninja / \n",
"22-Nature / \n",
"6-Animation / \n",
"6-Animation / 15-Cartoon / 1266-Death Note / \n",
"85-Combat / \n",
"12-Food / 684-Ice cream / \n",
"88-Machine / 415-Sewing / 997-Sewing machine / \n",
"60-Basketball / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"1-Vehicle / 249-Heavy equipment / 271-Tank / 817-Bulldozer / \n",
"7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"287-Human swimming / 329-Swimming pool / \n",
"0-Games / 8-Football / 76-Ball / 356-Star / \n",
"3-Concert / 30-Drummer / \n",
"3-Concert / 7-Musician / 16-Performance art / 30-Drummer / 39-Drums / \n",
"14-Guitar / 4518-Shakugan no Shana / \n",
"7-Musician / 14-Guitar / 44-Drums / 100-Electric guitar / \n",
"20-Trailer / \n",
"13-Musical ensemble / 21-Stadium / 49-School / 77-Arena / 79-American football / 203-Marching band / \n",
"20-Trailer / \n",
"5-Dance / 776-Kindergarten / \n",
"5-Dance / 2131-Flair bartending / \n",
"7-Musician / 13-Musical ensemble / 203-Marching band / 250-Parade / 376-Flute / \n",
"762-Angel / \n",
"75-Wrestling / 85-Combat / 117-Boxing / \n",
"88-Machine / 415-Sewing / 997-Sewing machine / \n",
"6-Animation / 816-Gundam / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 47-Personal computer / 145-Tablet computer / \n",
"101-Telephone / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 165-League of Legends / \n",
"102-Building / 170-Home improvement / 248-Hotel / 276-Room / 605-Cruise ship / 1402-Suite / \n",
"2-Video game / 43-Call of Duty / 194-Call of Duty: Modern Warfare 2 / 216-Call of Duty: Modern Warfare 3 / 315-Knife / \n",
"164-Beach / 219-Accordion / 233-Ocean / \n",
"3-Concert / 5-Dance / \n",
"1-Vehicle / 11-Motorsport / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / 719-Boeing 747 / \n",
"25-Toy / 51-Strategy video game / 264-Star Wars / \n",
"93-Comedy / \n",
"1-Vehicle / 11-Motorsport / 42-Fishing / 87-Boat / 1025-Personal water craft / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 114-Engine / 1763-Lotus Cars / \n",
"6-Animation / 290-Pony / 343-My Little Pony / \n",
"3-Concert / \n",
"0-Games / 8-Football / 76-Ball / 81-Athlete / 1182-Softball / \n",
"12-Food / 238-Nail / 827-Pump / 1510-Liquid / \n",
"253-Carnival / \n",
"12-Food / \n",
"7-Musician / 14-Guitar / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 184-Vegetable / 239-Roasting / 301-Meal / 500-Rice / 759-Icing / 2775-Broccoli / \n",
"5-Dance / 9-Music video / 115-Ballet / 130-Gymnastics / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 179-Kitchen / 226-Dessert / 534-Milk / 1725-Custard / \n",
"1975-Amiga / \n",
"2-Video game / 47-Personal computer / 228-Laptop / 414-Computer monitor / \n",
"25-Toy / 735-Kamen Rider Series / \n",
"8-Football / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 816-Gundam / \n",
"13-Musical ensemble / 203-Marching band / 253-Carnival / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 35-Minecraft / 245-BMW / \n",
"12-Food / 720-Wine / 3722-Wine tasting / \n",
"8-Football / 769-Red Bull / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 94-Dashcam / 164-Beach / 447-Sand / 901-Mitsubishi / 975-Dune / \n",
"5-Dance / 16-Performance art / \n",
"93-Comedy / 355-Comedian / \n",
"9-Music video / \n",
"18-Outdoor recreation / 267-Mountain / 347-Climbing / 614-Hiking / 2898-Canon EF lens mount / \n",
"31-Disc jockey / 133-Nightclub / \n",
"56-Hair / 106-Hairstyle / \n",
"6-Animation / 15-Cartoon / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / 1197-Hedgehog / \n",
"8-Football / \n",
"10-Animal / 12-Food / 943-Stove / \n",
"0-Games / 1991-Rappelz / \n",
"6-Animation / 89-Comics / 294-Batman / 630-Superman / 803-DC Comics / 3473-Nightwing / \n",
"3-Concert / 842-Harp / \n",
"0-Games / 310-The Sims / 328-Simulation video game / 1553-The Sims 4 / 1713-The Sims / \n",
"2-Video game / 1777-Ratchet / 1824-Clank / 1859-Ratchet & Clank / \n",
"12-Food / 348-Fruit / 718-Duck / 1319-Apple / \n",
"1-Vehicle / 50-Aircraft / \n",
"1-Vehicle / 4-Car / 70-Driving / 94-Dashcam / 245-BMW / 344-Coupé / 1233-BMW 3 Series / 2325-BMW 3 Series (E90) / \n",
"3-Concert / 5-Dance / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"6-Animation / 15-Cartoon / 89-Comics / \n",
"3-Concert / 559-Night / 1222-Fountain / \n",
"28-Fashion / 122-Weight training / 163-Running / 206-Shoe / 368-Sneakers / \n",
"9-Music video / 322-Mixtape / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 1031-Fisheye lens / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / \n",
"0-Games / 116-Pokémon / 186-Pokémon / \n",
"117-Boxing / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"12-Food / 26-Cooking / 32-Recipe / 367-Drink / 454-Sauce / 612-Beer / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 42-Fishing / 233-Ocean / 423-Coast / 1135-Dolphin / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 72-iPhone / 118-Grand Theft Auto V / \n",
"0-Games / 35-Minecraft / \n",
"1-Vehicle / 5-Dance / 27-Motorcycle / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"313-Book / \n",
"47-Personal computer / 228-Laptop / 414-Computer monitor / 835-Backpack / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 515-Dodge / 2366-Dodge Viper / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 75-Wrestling / \n",
"127-Xbox 360 / 724-Rock Band / 1703-Rock Band / \n",
"259-Fire / 350-Circus / 788-Juggling / \n",
"79-American football / \n",
"121-Photography / 155-Camera / 566-Digital camera / 629-Camera lens / 1117-Camcorder / 1855-Video camera / \n",
"1-Vehicle / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 220-Mountain bike / 284-Mountain biking / \n",
"6-Animation / 1187-Ragnarok Online / \n",
"1-Vehicle / 27-Motorcycle / \n",
"1-Vehicle / 139-Airport / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / 472-Boeing 737 / \n",
"6-Animation / 10-Animal / 15-Cartoon / \n",
"415-Sewing / \n",
"0-Games / 2-Video game / 10-Animal / 15-Cartoon / 132-Bird / 847-Angry Birds / \n",
"9-Music video / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / \n",
"0-Games / 1-Vehicle / 2-Video game / 19-PC game / 33-Weapon / 51-Strategy video game / 271-Tank / 409-World of Tanks / \n",
"341-Sketch comedy / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / \n",
"5-Dance / \n",
"0-Games / 60-Basketball / 76-Ball / 168-Basketball moves / 255-Slam dunk / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 55-Video game console / 199-Wii / 319-Robot / 443-Wii U / 1092-Game Boy Advance / \n",
"3-Concert / 5-Dance / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 76-Ball / 77-Arena / 81-Athlete / \n",
"1-Vehicle / \n",
"1063-Ink / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 380-Assassin's Creed / 1048-Assassin's Creed / 1522-Assassin's Creed II / \n",
"1-Vehicle / 4-Car / \n",
"1-Vehicle / 4-Car / 114-Engine / \n",
"1-Vehicle / 4-Car / 87-Boat / \n",
"0-Games / 2-Video game / 426-Resident Evil / 1238-Resident Evil 5 / 2278-Albert Wesker / \n",
"0-Games / 211-RuneScape / \n",
"10-Animal / 48-Pet / 71-Dog / 166-Cat / 346-Kitten / 3444-Claw / \n",
"1-Vehicle / 4-Car / 87-Boat / 208-Wood / \n",
"0-Games / 2-Video game / 90-Sports game / 180-FIFA 15 / \n",
"12-Food / 32-Recipe / 295-Chocolate / 1193-Orange / \n",
"28-Fashion / 56-Hair / 106-Hairstyle / 798-Shampoo / 919-Hair conditioner / 1320-Bangs / 1359-Hair iron / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 210-Cookware and bakeware / 239-Roasting / 771-Noodle / 1284-Frying / \n",
"60-Basketball / 168-Basketball moves / 255-Slam dunk / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 219-Accordion / 616-The Bible / 1019-Christ / \n",
"110-Album / \n",
"45-Cosmetics / 238-Nail / 300-Nail art / 304-Nail polish / 307-Hand / 324-Finger / 338-Manicure / \n",
"10-Animal / 166-Cat / 3944-Beanie / \n",
"209-University / \n",
"7-Musician / 14-Guitar / 24-String instrument / \n",
"5-Dance / \n",
"12-Food / 215-Eating / 1627-Shorts / \n",
"93-Comedy / \n",
"1146-Wing Chun / \n",
"1-Vehicle / 4-Car / 114-Engine / 299-Volkswagen Passenger Cars / 969-Volkswagen Beetle / \n",
"10-Animal / 166-Cat / \n",
"6-Animation / 25-Toy / 477-Transformers / 990-Optimus Prime / 4445-Transformers: Beast Wars / \n",
"20-Trailer / 66-Bollywood / \n",
"9-Music video / \n",
"10-Animal / 48-Pet / 132-Bird / 280-Wildlife / \n",
"359-Harry Potter (Literary Series) / \n",
"47-Personal computer / 228-Laptop / 525-Printer / 1063-Ink / 2634-Inkjet printing / \n",
"0-Games / 33-Weapon / 257-Counter-Strike / 702-Counter-Strike: Source / 826-Source Engine / 1513-Hitman / \n",
"113-House / 248-Hotel / 276-Room / 416-Apartment / 652-Bathroom / \n",
"18-Outdoor recreation / 347-Climbing / 828-Rock climbing / \n",
"46-Choir / \n",
"3-Concert / 9-Music video / \n",
"1083-Role-playing game / 3524-Tabletop role-playing game / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 2-Video game / 19-PC game / 89-Comics / 511-Mortal Kombat / 1392-Injustice: Gods Among Us / \n",
"9-Music video / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"1-Vehicle / 4-Car / 88-Machine / 114-Engine / 157-Jet aircraft / 1257-Turbine / 1529-Jet engine / \n",
"3-Concert / 7-Musician / \n",
"293-Slide show / \n",
"144-Amusement park / 321-Roller coaster / 334-Amusement ride / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / 413-PlayStation 2 / \n",
"1-Vehicle / 27-Motorcycle / \n",
"0-Games / 326-Metin2 / \n",
"131-Computer / 141-Microsoft Windows / \n",
"9-Music video / \n",
"0-Games / 51-Strategy video game / 1120-Tower defense / 2988-Bloons Tower Defense / 3354-Bloons / \n",
"3-Concert / 31-Disc jockey / \n",
"25-Toy / 140-LEGO / 884-Lego minifigure / \n",
"0-Games / 2-Video game / 6-Animation / 25-Toy / 96-Soldier / 940-Clay animation / \n",
"615-Poker / \n",
"6-Animation / 183-Manga / 750-Sailor Moon / 1342-Sailor Moon / \n",
"12-Food / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 47-Personal computer / 145-Tablet computer / 178-Samsung Galaxy / 487-Samsung Electronics / 1015-Samsung Galaxy Note series / 1727-Samsung Galaxy Note 3 / \n",
"15-Cartoon / 185-Dragon Ball / 244-Goku / \n",
"93-Comedy / \n",
"10-Animal / 12-Food / 22-Nature / 84-Snow / 120-Winter / 262-Ice / 1145-Winter storm / \n",
"6-Animation / 15-Cartoon / 183-Manga / \n",
"1-Vehicle / 4-Car / 1089-Wire / 3187-Spark plug / \n",
"335-Nintendo 3DS / 3901-Star Fox 64 / \n",
"3-Concert / 13-Musical ensemble / \n",
"38-Orchestra / \n",
"121-Photography / 1605-Second Life / \n",
"1-Vehicle / 2-Video game / 4-Car / 40-Road / 201-Grand Theft Auto: San Andreas / 297-Grand Theft Auto IV / 1425-Gears of War 2 / 3135-Call of Duty 3 / 3718-Pentium / \n",
"1345-Dean Winchester / 1801-Sam Winchester / \n",
"0-Games / 47-Personal computer / \n",
"1-Vehicle / 4-Car / 519-Recreational vehicle / \n",
"0-Games / 2-Video game / 6-Animation / 19-PC game / 123-Naruto / 591-Naruto: Ultimate Ninja / \n",
"10-Animal / 48-Pet / 106-Hairstyle / 132-Bird / 1341-Parakeet / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 70-Driving / 378-Drag racing / \n",
"429-Website / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 92-Radio-controlled model / 139-Airport / 149-Model aircraft / 157-Jet aircraft / 175-Airliner / 188-Airline / 205-Newscaster / 2206-Boeing 787 Dreamliner / \n",
"3-Concert / 16-Performance art / \n",
"22-Nature / 223-Weather / 296-Saxophone / 499-Cloud / 502-Sky / 1006-Tenor saxophone / \n",
"2233-Derek Shepherd / \n",
"31-Disc jockey / \n",
"3-Concert / \n",
"8-Football / \n",
"93-Comedy / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 150-Violin / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 378-Drag racing / 2133-Road racing / \n",
"85-Combat / \n",
"163-Running / \n",
"22-Nature / 283-Rock / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"1-Vehicle / 50-Aircraft / 114-Engine / 627-Clock / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 105-Motorcycling / 1036-Moped / \n",
"0-Games / 2-Video game / 51-Strategy video game / 2503-Age of Empires / 4125-Age of Empires II: The Age of Kings / \n",
"23-Mobile phone / 1820-Motorola Droid / \n",
"3-Concert / \n",
"113-House / 246-Furniture / 369-Resort / 416-Apartment / 580-Bedroom / \n",
"95-Talent show / \n",
"5-Dance / 7-Musician / 13-Musical ensemble / 203-Marching band / 1532-Military band / \n",
"5-Dance / 16-Performance art / \n",
"61-Art / 97-Drawing / 242-Paper / 415-Sewing / 457-Knitting / 546-Stitch / 2424-Greeting card / 3425-Christmas card / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / 265-Electronic keyboard / \n",
"67-Cycling / 69-Bicycle / 377-BMX bike / \n",
"2-Video game / 47-Personal computer / 111-PlayStation 3 / 194-Call of Duty: Modern Warfare 2 / 414-Computer monitor / \n",
"194-Call of Duty: Modern Warfare 2 / 2550-EarthBound / \n",
"0-Games / 8-Football / 21-Stadium / 79-American football / 81-Athlete / \n",
"2-Video game / 55-Video game console / 2142-Sega Saturn / \n",
"5-Dance / 16-Performance art / 458-Gold / \n",
"5-Dance / 7-Musician / 13-Musical ensemble / 16-Performance art / \n",
"1-Vehicle / 6-Animation / 87-Boat / 126-The Walt Disney Company / 308-Ship / 605-Cruise ship / \n",
"9-Music video / \n",
"12-Food / 534-Milk / \n",
"10-Animal / 48-Pet / 71-Dog / \n",
"85-Combat / \n",
"18-Outdoor recreation / 573-Paragliding / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 684-Ice cream / \n",
"18-Outdoor recreation / 347-Climbing / 3318-Abseiling / \n",
"6-Animation / 25-Toy / 140-LEGO / 319-Robot / \n",
"3-Concert / \n",
"31-Disc jockey / 861-Mixing console / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 51-Strategy video game / 171-Warcraft / 1691-Warcraft III: Reign of Chaos / \n",
"0-Games / 2-Video game / 637-Five Nights at Freddy's / 1069-Easter egg / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 272-iPod / 331-iPod touch / \n",
"18-Outdoor recreation / 22-Nature / 42-Fishing / 147-River / 268-Recreational fishing / \n",
"1-Vehicle / 4-Car / 40-Road / 94-Dashcam / 3496-Chevrolet Malibu / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"0-Games / 8-Football / \n",
"1443-Dance studio / \n",
"2-Video game / 55-Video game console / 127-Xbox 360 / 181-Xbox / 314-Xbox One / 386-Xbox / 677-Game controller / 1872-Xbox 360 Controller / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / \n",
"1-Vehicle / 8-Football / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 1125-God of War / 1432-Kratos / 1743-God of War / 2103-God of War III / \n",
"3-Concert / 98-Festival / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 74-Truck / 378-Drag racing / 761-Chevrolet Camaro / 1040-Street racing / 2317-Twin-turbo / \n",
"0-Games / 2-Video game / 240-Arcade game / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / \n",
"0-Games / 2-Video game / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / \n",
"20-Trailer / 365-Fireworks / 992-Rocket / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 159-World of Warcraft / 171-Warcraft / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 74-Truck / 92-Radio-controlled model / 167-Four-wheel drive / 198-Off-road vehicle / 202-Radio-controlled car / 234-Ford / 327-Mud / 360-Jeep / 565-Mud bogging / 935-Off-road racing / 3194-Ford F-350 / \n",
"978-Love song / \n",
"14-Guitar / 4518-Shakugan no Shana / \n",
"5-Dance / 49-School / \n",
"47-Personal computer / 131-Computer / 145-Tablet computer / 228-Laptop / 309-Computer hardware / 513-Computer keyboard / \n",
"208-Wood / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 19-PC game / 34-Action-adventure game / 387-Spider-Man / 1982-Venom / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / 1103-Gibson Les Paul / 3014-Gibson SG / \n",
"110-Album / \n",
"1-Vehicle / 4-Car / 62-Train / 428-Rail transport modelling / \n",
"18-Outdoor recreation / 22-Nature / 347-Climbing / 722-Waterfall / 828-Rock climbing / \n",
"1-Vehicle / 4-Car / 40-Road / 94-Dashcam / 223-Weather / 490-Rain / \n",
"88-Machine / 619-Manufacturing / 625-Factory / 812-Plastic / \n",
"12-Food / 215-Eating / 3385-Burrito / \n",
"56-Hair / 106-Hairstyle / 340-Afro-textured hair / \n",
"10-Animal / \n",
"88-Machine / 512-Printing / \n",
"209-University / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 98-Festival / 150-Violin / \n",
"10-Animal / 12-Food / 48-Pet / 52-Dish / 71-Dog / \n",
"78-Wedding / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 135-Drifting / 1519-Gran Turismo 6 / \n",
"2-Video game / 28-Fashion / 60-Basketball / 206-Shoe / 333-Nike; Inc. / 368-Sneakers / \n",
"0-Games / 8-Football / 21-Stadium / 358-Rugby football / \n",
"6-Animation / 15-Cartoon / 463-Cricket / \n",
"0-Games / 1000-AdventureQuest Worlds / \n",
"6-Animation / 15-Cartoon / 1555-Sword Art Online / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 212-Highway / 352-Need for Speed / 875-Need for Speed: Most Wanted / 1600-Need for Speed: World / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 102-Building / 129-Aviation / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 188-Airline / 254-Runway / \n",
"60-Basketball / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 42-Fishing / 91-Fish / 275-Diving / 288-Underwater / 381-Underwater diving / 407-Scuba diving / \n",
"10-Animal / 48-Pet / 166-Cat / 346-Kitten / \n",
"182-Puppy / \n",
"337-Advertising / 2032-Pepsi / \n",
"3-Concert / 68-Lighting / 594-Music festival / \n",
"2-Video game / 1087-Donkey Kong / 2970-Donkey Kong / 3950-Donkey Kong 64 / \n",
"3-Concert / 9-Music video / \n",
"20-Trailer / 359-Harry Potter (Literary Series) / 2001-Harry Potter and the Deathly Hallows / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 85-Combat / 380-Assassin's Creed / 1435-Ezio Auditore da Firenze / 1745-Assassin's Creed: Revelations / \n",
"0-Games / 2-Video game / 43-Call of Duty / 55-Video game console / 176-PlayStation 4 / 292-Handheld game console / 317-PlayStation / 422-Call of Duty: Ghosts / 682-PlayStation Vita / \n",
"1066-Aikido / \n",
"6-Animation / 15-Cartoon / \n",
"3-Concert / 68-Lighting / \n",
"31-Disc jockey / 133-Nightclub / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 230-Brass instrument / 296-Saxophone / \n",
"9-Music video / \n",
"116-Pokémon / 186-Pokémon / 1387-Pokémon Battle Revolution / \n",
"36-Piano / 53-Keyboard / 104-Pianist / \n",
"45-Cosmetics / 56-Hair / 173-Eye shadow / 193-Eye liner / 195-Mascara / 235-Lipstick / 282-Skin / 291-Rouge / 385-Face / 436-Concealer / \n",
"1345-Dean Winchester / \n",
"141-Microsoft Windows / 513-Computer keyboard / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 178-Samsung Galaxy / \n",
"20-Trailer / \n",
"0-Games / 2-Video game / 756-Tibia / 4097-Nollie / \n",
"10-Animal / 132-Bird / 718-Duck / 1947-Goose / \n",
"126-The Walt Disney Company / 1106-Character / \n",
"16-Performance art / 93-Comedy / 355-Comedian / 2463-Smile / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 2-Video game / 19-PC game / 1049-Collectible card game / \n",
"10-Animal / 48-Pet / 182-Puppy / \n",
"5-Dance / 16-Performance art / 200-Doll / \n",
"5-Dance / 16-Performance art / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 94-Dashcam / \n",
"0-Games / 2-Video game / 55-Video game console / 176-PlayStation 4 / 317-PlayStation / 1482-PlayStation Network / \n",
"47-Personal computer / 131-Computer / 497-Macintosh / \n",
"12-Food / 26-Cooking / 4138-Chorizo / \n",
"0-Games / 2-Video game / 25-Toy / 140-LEGO / 294-Batman / 2440-Lego Batman: The Videogame / \n",
"102-Building / 113-House / 353-Wall / 433-Construction / 488-Door / 1314-Garage / 3457-Garage door / \n",
"6-Animation / 15-Cartoon / 1866-Katara / 3734-Aang / \n",
"31-Disc jockey / \n",
"315-Knife / 442-Bread / 2017-Sharpening / 3880-Knife sharpening / \n",
"13-Musical ensemble / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 169-Rallying / \n",
"12-Food / 26-Cooking / 32-Recipe / 391-Chicken meat / 1707-Microwave oven / \n",
"9-Music video / \n",
"3-Concert / 7-Musician / 16-Performance art / 30-Drummer / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 668-Volkswagen Golf / 2517-Chevrolet Opala / \n",
"364-Sitcom / \n",
"36-Piano / 53-Keyboard / 446-Sheet music / 592-Synthesia / \n",
"3-Concert / 68-Lighting / \n",
"9-Music video / \n",
"14-Guitar / 100-Electric guitar / \n",
"5-Dance / 7-Musician / 78-Wedding / \n",
"61-Art / 148-Painting / 628-Portrait / \n",
"0-Games / 2-Video game / 47-Personal computer / 908-Video card / 1580-Crysis 2 / \n",
"1-Vehicle / 4-Car / 6-Animation / 15-Cartoon / 25-Toy / 74-Truck / 88-Machine / 1307-Police car / 1418-Model car / 2701-Tow truck / \n",
"6-Animation / 15-Cartoon / 567-Compact disc / 1492-Persona / \n",
"23-Mobile phone / 29-Smartphone / \n",
"66-Bollywood / \n",
"22-Nature / 164-Beach / 233-Ocean / 423-Coast / \n",
"1-Vehicle / 4-Car / 531-Hatchback / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 81-Athlete / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 1144-Castlevania / 3768-Castlevania: Lords of Shadow / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 134-Call of Duty: Black Ops / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"98-Festival / \n",
"707-Roof / \n",
"61-Art / \n",
"2716-The Hunger Games / 4616-Catching Fire / \n",
"28-Fashion / 793-Shirt / \n",
"1-Vehicle / \n",
"3-Concert / \n",
"9-Music video / \n",
"2840-Engraving / 2967-Printmaking / \n",
"6-Animation / 25-Toy / 477-Transformers / 990-Optimus Prime / \n",
"3-Concert / 68-Lighting / \n",
"1243-Mirror / \n",
"379-Touhou Project / \n",
"242-Paper / 370-Origami / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / \n",
"66-Bollywood / \n",
"66-Bollywood / \n",
"12-Food / 4609-Pepper spray / \n",
"1-Vehicle / 4-Car / 74-Truck / 375-Pickup truck / 3309-Toyota Tundra / \n",
"4017-Television film / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 554-iPhone 4 / 1155-iPhone 3GS / \n",
"0-Games / 2-Video game / 199-Wii / 2502-GoldenEye 007 / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 297-Grand Theft Auto IV / 658-Grand Theft Auto: The Lost and Damned / 1923-Niko Bellic / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 380-Assassin's Creed / 1140-Assassin's Creed III / \n",
"576-Walt Disney World / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / \n",
"44-Drums / 3215-Cornet / \n",
"10-Animal / 71-Dog / \n",
"5-Dance / 16-Performance art / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / 336-Scooter / \n",
"2342-Jack Sparrow / \n",
"21-Stadium / \n",
"8-Football / \n",
"6-Animation / 183-Manga / 269-Comic book / \n",
"6-Animation / 3163-Tugboat / \n",
"46-Choir / \n",
"1-Vehicle / 4-Car / 234-Ford / \n",
"56-Hair / 106-Hairstyle / \n",
"1-Vehicle / 49-School / 286-Bus / 2199-School bus / \n",
"0-Games / 1-Vehicle / 50-Aircraft / 139-Airport / 151-Landing / 188-Airline / 923-Boeing 777 / \n",
"10-Animal / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 23-Mobile phone / 29-Smartphone / 72-iPhone / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / 331-iPod touch / \n",
"5-Dance / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 935-Off-road racing / 2734-Monster Energy / 4003-Baja 1000 / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"1-Vehicle / 4-Car / 1624-Minivan / 2446-Used car / \n",
"102-Building / 113-House / 310-The Sims / 328-Simulation video game / 564-The Sims 3 / \n",
"0-Games / 19-PC game / 35-Minecraft / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 447-Sand / 751-Desert / 975-Dune / \n",
"0-Games / 8-Football / 21-Stadium / 77-Arena / \n",
"536-Glass / 1226-Metalworking / 2855-Vase / \n",
"2848-12-inch single / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / 3454-Paiste / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 428-Rail transport modelling / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 1024-Culinary art / \n",
"0-Games / 2-Video game / 2592-DualShock / 3862-Toph Beifong / \n",
"3-Concert / 5-Dance / 98-Festival / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / 44-Drums / \n",
"0-Games / 1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 70-Driving / \n",
"110-Album / \n",
"0-Games / 21-Stadium / 60-Basketball / 77-Arena / \n",
"0-Games / 2-Video game / 310-The Sims / 328-Simulation video game / 1553-The Sims 4 / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / 203-Marching band / 1675-Drum and bugle corps / \n",
"18-Outdoor recreation / 33-Weapon / 217-Hunting / \n",
"0-Games / 2-Video game / 1789-Slender: The Eight Pages / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 44-Drums / 78-Wedding / \n",
"260-Prayer / \n",
"3-Concert / \n",
"5-Dance / \n",
"31-Disc jockey / \n",
"14-Guitar / \n",
"31-Disc jockey / \n",
"12-Food / 1227-Sushi / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 135-Drifting / \n",
"205-Newscaster / \n",
"7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"31-Disc jockey / 342-Windows Media Video / \n",
"6-Animation / 185-Dragon Ball / 244-Goku / 876-Freeza / \n",
"10-Animal / 12-Food / 26-Cooking / 32-Recipe / 91-Fish / 215-Eating / 239-Roasting / 893-Fish as food / 3825-Tilapia / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 19-PC game / 33-Weapon / 734-DayZ / 3202-Gesture / \n",
"250-Parade / \n",
"0-Games / 2-Video game / 25-Toy / 140-LEGO / 264-Star Wars / 852-Lego Star Wars / \n",
"0-Games / 2-Video game / 35-Minecraft / 127-Xbox 360 / 181-Xbox / \n",
"8-Football / \n",
"6-Animation / 15-Cartoon / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 345-Loudspeaker / 349-Chevrolet / 761-Chevrolet Camaro / \n",
"3-Concert / 5-Dance / 16-Performance art / 49-School / 95-Talent show / \n",
"6-Animation / 4271-Inflatable castle / \n",
"3-Concert / \n",
"83-Skateboarding / 108-Skateboard / 555-Longboard / 667-Longboarding / \n",
"0-Games / 1083-Role-playing game / \n",
"6-Animation / 15-Cartoon / 396-One Piece / \n",
"14-Guitar / \n",
"6-Animation / 15-Cartoon / 123-Naruto / 622-Sakura Haruno / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 570-Road bicycle racing / \n",
"38-Orchestra / \n",
"36-Piano / 53-Keyboard / 104-Pianist / \n",
"12-Food / 86-Plant / 1023-Rose / \n",
"3-Concert / 7-Musician / \n",
"9-Music video / \n",
"5-Dance / 49-School / 95-Talent show / 191-Cheerleading / \n",
"7-Musician / 30-Drummer / 39-Drums / 44-Drums / \n",
"0-Games / 240-Arcade game / 1440-Pinball / \n",
"10-Animal / 80-Horse / \n",
"10-Animal / 48-Pet / 132-Bird / 598-Parrot / 1341-Parakeet / 2024-Cockatoo / \n",
"75-Wrestling / \n",
"0-Games / 8-Football / 65-Kick / 76-Ball / \n",
"130-Gymnastics / 1558-Hula hoop / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 33-Weapon / \n",
"3-Concert / 38-Orchestra / \n",
"6-Animation / 123-Naruto / 622-Sakura Haruno / \n",
"6-Animation / 25-Toy / 35-Minecraft / 140-LEGO / 3899-Lego Minecraft / \n",
"3-Concert / 1442-Tabla / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"2-Video game / 20-Trailer / \n",
"5-Dance / 115-Ballet / \n",
"159-World of Warcraft / 171-Warcraft / \n",
"1-Vehicle / 4-Car / 484-Subwoofer / \n",
"12-Food / 26-Cooking / 32-Recipe / 226-Dessert / 232-Cake / 236-Baking / 795-Cupcake / 1192-Hello Kitty / \n",
"455-Unidentified flying object / \n",
"72-iPhone / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 96-Soldier / 217-Hunting / 601-Shark / 1543-Crysis / \n",
"1-Vehicle / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 393-Unmanned aerial vehicle / 604-Quadcopter / 2025-Multirotor / \n",
"88-Machine / 208-Wood / 400-Woodturning / 3061-Cylinder / \n",
"9-Music video / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"238-Nail / 300-Nail art / 304-Nail polish / 307-Hand / 324-Finger / 338-Manicure / \n",
"45-Cosmetics / 173-Eye shadow / \n",
"56-Hair / 106-Hairstyle / 164-Beach / 329-Swimming pool / 405-Braid / \n",
"110-Album / \n",
"546-Stitch / \n",
"1-Vehicle / 4-Car / 84-Snow / 94-Dashcam / 120-Winter / 262-Ice / 680-Subaru / \n",
"99-Christmas / 3425-Christmas card / \n",
"5-Dance / 49-School / 95-Talent show / 130-Gymnastics / 191-Cheerleading / 306-High school / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / \n",
"2454-Notebook / \n",
"0-Games / 2-Video game / 6-Animation / 111-PlayStation 3 / 2368-Marvel Universe / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 39-Drums / \n",
"0-Games / 19-PC game / 51-Strategy video game / 382-Dota 2 / \n",
"3-Concert / 5-Dance / 888-Living room / 1281-Couch / \n",
"8-Football / \n",
"3-Concert / 7-Musician / \n",
"3-Concert / 9-Music video / \n",
"66-Bollywood / \n",
"0-Games / 791-Magic: The Gathering / \n",
"83-Skateboarding / 108-Skateboard / 2256-Bearing / \n",
"3-Concert / 68-Lighting / \n",
"1-Vehicle / 27-Motorcycle / 1282-Honda CBR series / \n",
"9-Music video / \n",
"5-Dance / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 927-Supermoto / 1036-Moped / \n",
"385-Face / 780-Fat / \n",
"8-Football / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 57-Race track / 74-Truck / 854-Diesel engine / \n",
"1791-Stargate / \n",
"0-Games / 12-Food / 578-Cookie / \n",
"12-Food / 367-Drink / 1347-Coffeehouse / \n",
"0-Games / 256-Tennis / 1337-Serve / 2565-Soft tennis / \n",
"125-Television / \n",
"9-Music video / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / \n",
"0-Games / 2-Video game / 20-Trailer / 55-Video game console / 176-PlayStation 4 / 292-Handheld game console / 317-PlayStation / 682-PlayStation Vita / \n",
"681-Leather / 1525-Wallet / 2669-Leather crafting / \n",
"0-Games / 2-Video game / 33-Weapon / 850-Left 4 Dead / 963-Left 4 Dead 2 / \n",
"36-Piano / 53-Keyboard / 104-Pianist / \n",
"122-Weight training / 204-Gym / 214-Muscle / 1430-Pull-up / \n",
"10-Animal / 48-Pet / 71-Dog / 3173-Great Dane / \n",
"56-Hair / 106-Hairstyle / 200-Doll / \n",
"6-Animation / 15-Cartoon / 116-Pokémon / 1104-Ash Ketchum / 3347-Misty / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / 1240-Mambo / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / \n",
"0-Games / 2-Video game / 35-Minecraft / 127-Xbox 360 / 181-Xbox / 321-Roller coaster / \n",
"66-Bollywood / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 57-Race track / 169-Rallying / \n",
"3-Concert / \n",
"9-Music video / \n",
"6-Animation / 379-Touhou Project / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"2-Video game / 20-Trailer / \n",
"5-Dance / \n",
"23-Mobile phone / 29-Smartphone / 178-Samsung Galaxy / 1287-Samsung Galaxy Tab series / 1390-Samsung Galaxy S II / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / 719-Boeing 747 / 2790-British Airways / \n",
"5-Dance / 256-Tennis / \n",
"0-Games / 2-Video game / 51-Strategy video game / 127-Xbox 360 / 181-Xbox / \n",
"0-Games / 8-Football / 1016-Glove / \n",
"0-Games / 19-PC game / 35-Minecraft / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 40-Road / 105-Motorcycling / 1414-Gasoline / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 493-Rim / \n",
"67-Cycling / 69-Bicycle / 83-Skateboarding / 221-Skatepark / 377-BMX bike / \n",
"7-Musician / 22-Nature / 46-Choir / 86-Plant / \n",
"1604-Satan / \n",
"0-Games / 2-Video game / 8-Football / 21-Stadium / 73-Ball / 79-American football / 90-Sports game / 561-Madden NFL / \n",
"9-Music video / \n",
"121-Photography / 155-Camera / 566-Digital camera / 629-Camera lens / 721-Digital SLR / \n",
"0-Games / 49-School / 60-Basketball / 168-Basketball moves / 306-High school / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 274-Meat / 688-Beef / 752-Pasta / 1230-Italian food / 2562-Meatball / \n",
"5-Dance / 259-Fire / 350-Circus / 788-Juggling / \n",
"0-Games / 59-Winter sport / 81-Athlete / 112-Ice skating / 142-Hockey / 312-Ice rink / \n",
"9-Music video / \n",
"1-Vehicle / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 220-Mountain bike / 645-Road bicycle / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 123-Naruto / 285-Sasuke Uchiha / 591-Naruto: Ultimate Ninja / \n",
"85-Combat / 117-Boxing / \n",
"256-Tennis / 974-Racket / \n",
"3-Concert / 16-Performance art / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"3-Concert / 9-Music video / \n",
"5-Dance / 49-School / \n",
"1-Vehicle / 27-Motorcycle / \n",
"1-Vehicle / 25-Toy / 62-Train / 3396-Ressha Sentai ToQger / \n",
"28-Fashion / \n",
"1082-Knot / 1369-Necktie / \n",
"28-Fashion / 45-Cosmetics / 235-Lipstick / \n",
"380-Assassin's Creed / 1522-Assassin's Creed II / \n",
"22-Nature / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"0-Games / 2-Video game / 8-Football / 21-Stadium / 54-Highlight film / 73-Ball / 90-Sports game / 453-Pro Evolution Soccer / \n",
"9-Music video / \n",
"9-Music video / 308-Ship / \n",
"0-Games / 60-Basketball / 168-Basketball moves / 255-Slam dunk / \n",
"754-Gramophone record / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 553-Sony Xperia / 2770-Sony Xperia Z1 / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 226-Dessert / \n",
"365-Fireworks / \n",
"75-Wrestling / 85-Combat / \n",
"260-Prayer / \n",
"75-Wrestling / \n",
"0-Games / 1000-AdventureQuest Worlds / 1178-Quest / \n",
"2-Video game / 192-Battlefield / 374-Battlefield 3 / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 151-Landing / 157-Jet aircraft / 187-Takeoff / 388-Fighter aircraft / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / \n",
"1411-ESPN / \n",
"67-Cycling / 69-Bicycle / \n",
"9-Music video / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"6-Animation / 15-Cartoon / 2948-Dogfight / 2974-Macross / \n",
"0-Games / 8-Football / \n",
"10-Animal / 18-Outdoor recreation / 42-Fishing / 91-Fish / 147-River / 268-Recreational fishing / 478-Fishing rod / 698-Fisherman / 844-Bass / 3292-Rock fishing / \n",
"5-Dance / 95-Talent show / 263-Light / \n",
"1-Vehicle / 4-Car / \n",
"28-Fashion / 390-Model / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 929-Aerobatics / \n",
"6-Animation / 15-Cartoon / \n",
"18-Outdoor recreation / 22-Nature / 164-Beach / 223-Weather / 233-Ocean / 423-Coast / 502-Sky / \n",
"25-Toy / 35-Minecraft / 140-LEGO / 264-Star Wars / 852-Lego Star Wars / 1452-BIONICLE / 1478-Lego Ninjago / 3181-Lego Legends of Chima / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 651-Stock car racing / \n",
"162-Medicine / 209-University / \n",
"3-Concert / 7-Musician / 14-Guitar / 24-String instrument / 39-Drums / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 74-Truck / 167-Four-wheel drive / 198-Off-road vehicle / 327-Mud / 375-Pickup truck / 565-Mud bogging / 4251-Chevrolet K5 Blazer / \n",
"567-Compact disc / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"3-Concert / 133-Nightclub / \n",
"1-Vehicle / 11-Motorsport / 124-Tractor / 156-Agriculture / 252-Tractor pulling / 459-Plough / \n",
"28-Fashion / \n",
"5-Dance / 95-Talent show / \n",
"1-Vehicle / 4-Car / 1163-Carpet / \n",
"8-Football / \n",
"9-Music video / \n",
"219-Accordion / 655-Diatonic button accordion / \n",
"0-Games / 33-Weapon / 1394-Alliance of Valiant Arms / \n",
"0-Games / 6-Animation / 2750-Mii / \n",
"78-Wedding / 190-Bride / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 40-Road / \n",
"3-Concert / 5-Dance / 16-Performance art / 46-Choir / \n",
"2224-Sherlock Holmes / \n",
"12-Food / 86-Plant / \n",
"0-Games / 19-PC game / 35-Minecraft / \n",
"3-Concert / 5-Dance / 28-Fashion / 98-Festival / 785-Jeans / \n",
"432-Kingdom Hearts / 1730-Roxas / 3635-Kingdom Hearts 358/2 Days / \n",
"5-Dance / 115-Ballet / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 76-Ball / 81-Athlete / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 135-Drifting / \n",
"3-Concert / 36-Piano / 53-Keyboard / 107-Musical keyboard / \n",
"1-Vehicle / 4-Car / 70-Driving / 484-Subwoofer / \n",
"3-Concert / 16-Performance art / \n",
"59-Winter sport / 112-Ice skating / 142-Hockey / \n",
"3-Concert / 7-Musician / 44-Drums / \n",
"322-Mixtape / \n",
"12-Food / 1088-Infomercial / 1162-Dietary supplement / \n",
"1-Vehicle / 62-Train / 119-Locomotive / 143-Railroad car / 161-Train station / 2824-Diesel locomotive / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / \n",
"12-Food / \n",
"0-Games / 2-Video game / 951-Teenage Mutant Ninja Turtles / \n",
"2080-Cup / \n",
"10-Animal / 18-Outdoor recreation / 42-Fishing / 91-Fish / 147-River / 225-Lake / 510-Fishing lure / 896-Largemouth bass / \n",
"25-Toy / 148-Painting / 303-Paint / 873-Acrylic paint / \n",
"117-Boxing / \n",
"86-Plant / 136-Gardening / 154-Wheel / 158-Tire / 279-Garden / \n",
"273-News program / \n",
"1-Vehicle / 1879-Hot air balloon / \n",
"122-Weight training / 125-Television / \n",
"196-Eye / 2120-Tears / \n",
"1-Vehicle / 4-Car / 25-Toy / 27-Motorcycle / 40-Road / 105-Motorcycling / 154-Wheel / 207-Exhaust system / 1380-Kawasaki motorcycles / 1951-Sport bike / \n",
"9-Music video / 84-Snow / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 3839-Volkswagen Golf Mk5 / \n",
"402-Jewellery / 458-Gold / 521-Metal / 731-Coin / 912-Silver / 1001-Gemstone / 1300-Earrings / \n",
"31-Disc jockey / \n",
"3-Concert / \n",
"59-Winter sport / 83-Skateboarding / 112-Ice skating / 703-Roller skating / 1783-Roller skates / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"0-Games / 2-Video game / 411-MapleStory / \n",
"8-Football / \n",
"75-Wrestling / 85-Combat / \n",
"1909-Analog synthesizer / \n",
"1-Vehicle / 4-Car / 1134-Taxicab / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 111-PlayStation 3 / 118-Grand Theft Auto V / \n",
"3-Concert / \n",
"12-Food / 26-Cooking / 52-Dish / 99-Christmas / \n",
"0-Games / 85-Combat / 211-RuneScape / \n",
"1-Vehicle / 50-Aircraft / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 266-Helicopter / 392-Radio-controlled helicopter / 393-Unmanned aerial vehicle / 455-Unidentified flying object / 604-Quadcopter / \n",
"144-Amusement park / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 77-Arena / 1247-Penalty kick / \n",
"28-Fashion / 676-Handbag / 835-Backpack / \n",
"1054-Vacuum cleaner / \n",
"0-Games / 2-Video game / 6-Animation / 20-Trailer / 111-PlayStation 3 / 176-PlayStation 4 / 849-Tales / 3883-Tales of Xillia / \n",
"5-Dance / 16-Performance art / 350-Circus / \n",
"28-Fashion / 153-Dress / \n",
"0-Games / 43-Call of Duty / \n",
"0-Games / 2-Video game / 116-Pokémon / 186-Pokémon / \n",
"99-Christmas / 293-Slide show / 558-Santa Claus / \n",
"5-Dance / 13-Musical ensemble / 250-Parade / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 81-Athlete / \n",
"0-Games / 2-Video game / 241-Fighting game / 311-Street Fighter / 461-Street Fighter IV / 496-Super Street Fighter IV / 1732-Oni / \n",
"0-Games / 1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / \n",
"89-Comics / 185-Dragon Ball / 244-Goku / 418-Gohan / \n",
"28-Fashion / 282-Skin / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 39-Drums / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 111-PlayStation 3 / 422-Call of Duty: Ghosts / \n",
"2-Video game / 6-Animation / 15-Cartoon / 20-Trailer / 123-Naruto / 591-Naruto: Ultimate Ninja / \n",
"0-Games / 51-Strategy video game / 831-Plants vs. Zombies / \n",
"9-Music video / 110-Album / \n",
"5-Dance / 16-Performance art / \n",
"20-Trailer / 434-Ninja / 951-Teenage Mutant Ninja Turtles / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 197-Trail / 220-Mountain bike / 284-Mountain biking / 645-Road bicycle / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / \n",
"33-Weapon / 992-Rocket / \n",
"9-Music video / \n",
"7-Musician / 44-Drums / 410-Teacher / 1442-Tabla / \n",
"8-Football / 259-Fire / \n",
"1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 118-Grand Theft Auto V / 158-Tire / \n",
"0-Games / 49-School / 60-Basketball / \n",
"45-Cosmetics / 56-Hair / \n",
"337-Advertising / 512-Printing / 1757-Screen printing / \n",
"131-Computer / 141-Microsoft Windows / \n",
"3-Concert / \n",
"49-School / 163-Running / \n",
"6-Animation / 712-Outer space / \n",
"0-Games / 2-Video game / 19-PC game / 96-Soldier / 192-Battlefield / 374-Battlefield 3 / \n",
"0-Games / 59-Winter sport / 77-Arena / 81-Athlete / 112-Ice skating / 142-Hockey / 312-Ice rink / \n",
"5-Dance / 31-Disc jockey / \n",
"5-Dance / 49-School / \n",
"18-Outdoor recreation / 287-Human swimming / 710-Handball / 2360-Water polo / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 47-Personal computer / 131-Computer / 141-Microsoft Windows / \n",
"2493-The Hobbit / \n",
"3-Concert / 7-Musician / \n",
"10-Animal / 80-Horse / 247-Stallion / 289-Dressage / \n",
"5-Dance / 16-Performance art / \n",
"5-Dance / 31-Disc jockey / \n",
"2-Video game / \n",
"5-Dance / \n",
"18-Outdoor recreation / 59-Winter sport / 84-Snow / 120-Winter / 177-Skiing / 305-Ski / \n",
"5-Dance / \n",
"31-Disc jockey / \n",
"0-Games / 2-Video game / 116-Pokémon / 186-Pokémon / 1204-Pokémon Ruby and Sapphire / 2933-Pokémon Trainer / \n",
"0-Games / 8-Football / \n",
"3-Concert / \n",
"0-Games / 49-School / 54-Highlight film / 60-Basketball / 76-Ball / 1101-Point guard / \n",
"0-Games / 8-Football / 65-Kick / 73-Ball / 76-Ball / \n",
"2-Video game / 25-Toy / 140-LEGO / 884-Lego minifigure / 1452-BIONICLE / \n",
"7-Musician / 13-Musical ensemble / 49-School / 203-Marching band / \n",
"3-Concert / 5-Dance / 133-Nightclub / \n",
"5-Dance / \n",
"219-Accordion / \n",
"1-Vehicle / 4-Car / 70-Driving / 114-Engine / \n",
"0-Games / \n",
"25-Toy / 1154-Anpanman / \n",
"10-Animal / 132-Bird / 404-Livestock / 424-Chicken / 1136-Poultry / 1428-Rooster / \n",
"9-Music video / \n",
"162-Medicine / \n",
"1-Vehicle / 4-Car / 154-Wheel / 158-Tire / 327-Mud / 3828-Amphibious ATV / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 918-Brake / 953-Gran Turismo 5 / 983-Lexus / \n",
"14-Guitar / 24-String instrument / \n",
"28-Fashion / 206-Shoe / 681-Leather / 757-Boot / \n",
"28-Fashion / 45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 154-Wheel / \n",
"3-Concert / 5-Dance / 16-Performance art / 1546-Line / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 118-Grand Theft Auto V / 201-Grand Theft Auto: San Andreas / 297-Grand Theft Auto IV / 413-PlayStation 2 / 858-Carl Johnson / \n",
"3-Concert / \n",
"3-Concert / 7-Musician / 14-Guitar / 361-Flamenco / \n",
"5-Dance / \n",
"9-Music video / \n",
"10-Animal / 15-Cartoon / 48-Pet / 71-Dog / 166-Cat / 346-Kitten / 2997-Talking Tom and Friends / \n",
"0-Games / 8-Football / 21-Stadium / 79-American football / 81-Athlete / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"1106-Character / 2368-Marvel Universe / \n",
"28-Fashion / 206-Shoe / 333-Nike; Inc. / 368-Sneakers / 763-Snake / 4171-Air Force 1 / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 149-Model aircraft / 693-Sword / 778-Gliding / 1317-Glider / \n",
"6-Animation / 15-Cartoon / \n",
"28-Fashion / 56-Hair / 1359-Hair iron / 1827-Hair straightening / \n",
"12-Food / 25-Toy / 42-Fishing / 1154-Anpanman / \n",
"8-Football / \n",
"6-Animation / 9-Music video / 15-Cartoon / \n",
"1-Vehicle / 4-Car / 40-Road / 259-Fire / 371-Firefighter / \n",
"93-Comedy / 355-Comedian / \n",
"6-Animation / 61-Art / 97-Drawing / 148-Painting / 628-Portrait / \n",
"1149-Castle / 2033-Princess Peach / 2708-Paper Mario / \n",
"10-Animal / 48-Pet / 71-Dog / \n",
"89-Comics / 781-Pachinko / \n",
"0-Games / 2-Video game / 33-Weapon / 192-Battlefield / 383-Battlefield 4 / \n",
"3-Concert / 30-Drummer / \n",
"5-Dance / 95-Talent show / \n",
"1288-Marvel vs. Capcom / 1505-Ultimate Marvel vs. Capcom 3 / 3109-Clint Barton / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 46-Choir / 49-School / \n",
"250-Parade / \n",
"3-Concert / \n",
"1-Vehicle / 11-Motorsport / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / 318-Church / \n",
"7-Musician / 46-Choir / 307-Hand / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / 594-Music festival / \n",
"24-String instrument / 842-Harp / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 172-Motocross / \n",
"223-Weather / 490-Rain / 499-Cloud / 502-Sky / 958-Thunderstorm / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / 78-Wedding / 209-University / \n",
"22-Nature / 42-Fishing / 275-Diving / 288-Underwater / \n",
"1-Vehicle / 50-Aircraft / 139-Airport / 151-Landing / \n",
"0-Games / 148-Painting / 878-Composer / \n",
"79-American football / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"9-Music video / \n",
"0-Games / 165-League of Legends / \n",
"6-Animation / 455-Unidentified flying object / \n",
"25-Toy / 140-LEGO / 884-Lego minifigure / \n",
"0-Games / 1109-Counter-Strike Online / \n",
"8-Football / \n",
"238-Nail / 300-Nail art / \n",
"9-Music video / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 143-Railroad car / \n",
"10-Animal / 25-Toy / 329-Swimming pool / 524-Littlest Pet Shop / \n",
"1-Vehicle / 4-Car / 74-Truck / 158-Tire / 234-Ford / 484-Subwoofer / 3684-Ford Bronco / \n",
"0-Games / 831-Plants vs. Zombies / 1185-Plasticine / \n",
"2-Video game / 6-Animation / \n",
"59-Winter sport / 112-Ice skating / 316-Figure skating / 632-Ice dancing / \n",
"359-Harry Potter (Literary Series) / 1436-Draco Malfoy / 1856-Hogwarts School of Witchcraft and Wizardry / \n",
"0-Games / 462-Table / 544-Table tennis / 1615-Pong / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 802-Google Nexus / \n",
"45-Cosmetics / 193-Eye liner / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / \n",
"8-Football / 259-Fire / \n",
"0-Games / 2-Video game / 3747-Wario / \n",
"0-Games / 2-Video game / 411-MapleStory / \n",
"0-Games / 2-Video game / 20-Trailer / 127-Xbox 360 / 181-Xbox / 1425-Gears of War 2 / \n",
"3-Concert / \n",
"6-Animation / 15-Cartoon / 116-Pokémon / 1104-Ash Ketchum / 3347-Misty / \n",
"28-Fashion / 206-Shoe / 1300-Earrings / \n",
"794-Bridge / \n",
"117-Boxing / \n",
"9-Music video / 390-Model / \n",
"6-Animation / 183-Manga / 2065-Tsubasa: Reservoir Chronicle / \n",
"1-Vehicle / 4-Car / 20-Trailer / 245-BMW / \n",
"0-Games / 2-Video game / 55-Video game console / 2034-Dreamcast / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 230-Brass instrument / 427-Trumpet / \n",
"567-Compact disc / \n",
"98-Festival / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"10-Animal / 125-Television / 384-Horse racing / \n",
"75-Wrestling / 1042-Jumbotron / \n",
"6-Animation / 89-Comics / 413-PlayStation 2 / \n",
"49-School / \n",
"863-Body piercing / \n",
"3151-Kermit the Frog / \n",
"1-Vehicle / 4-Car / 683-Renault / 1242-Car wash / 1678-Foam / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 64-Transport / 286-Bus / \n",
"1-Vehicle / 4-Car / 114-Engine / 360-Jeep / 982-V8 engine / 1406-Jeep Cherokee / 1984-Jeep Grand Cherokee / \n",
"8-Football / \n",
"3-Concert / 16-Performance art / \n",
"516-Headphones / \n",
"117-Boxing / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 94-Dashcam / 212-Highway / 800-Mountain pass / \n",
"12-Food / 32-Recipe / 282-Skin / 385-Face / 641-Mask / 784-Honey / 3404-Oatmeal / \n",
"20-Trailer / \n",
"122-Weight training / 214-Muscle / 390-Model / \n",
"18-Outdoor recreation / 22-Nature / 267-Mountain / 283-Rock / 614-Hiking / 1808-National park / \n",
"7-Musician / 30-Drummer / 39-Drums / 44-Drums / 1033-Drill / \n",
"170-Home improvement / 208-Wood / 246-Furniture / 462-Table / \n",
"56-Hair / 607-Lace / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / 64-Transport / 105-Motorcycling / 336-Scooter / 482-Asphalt / \n",
"0-Games / 2-Video game / 89-Comics / \n",
"28-Fashion / 469-Bag / 676-Handbag / 681-Leather / \n",
"2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 189-iPad / 272-iPod / 331-iPod touch / 554-iPhone 4 / 675-iPhone 4S / 1009-iPhone 3G / 1155-iPhone 3GS / \n",
"2-Video game / 6-Animation / 477-Transformers / 2186-Megatron / 4445-Transformers: Beast Wars / \n",
"31-Disc jockey / 133-Nightclub / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 422-Call of Duty: Ghosts / \n",
"3-Concert / \n",
"9-Music video / \n",
"1995-Hammer / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 553-Sony Xperia / 3019-Sony Ericsson Xperia X10 / \n",
"5-Dance / 747-Retail / 1328-Grocery store / \n",
"0-Games / 2-Video game / 2237-Kantai Collection / \n",
"402-Jewellery / 457-Knitting / 829-Bead / 1082-Knot / 2961-Button / \n",
"8-Football / \n",
"23-Mobile phone / 29-Smartphone / 178-Samsung Galaxy / 1727-Samsung Galaxy Note 3 / \n",
"66-Bollywood / \n",
"3-Concert / \n",
"5-Dance / 78-Wedding / 190-Bride / 421-Gown / \n",
"0-Games / 2-Video game / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / 4019-Sonic the Hedgehog CD / \n",
"233-Ocean / 248-Hotel / \n",
"3-Concert / \n",
"791-Magic: The Gathering / \n",
"61-Art / 148-Painting / 303-Paint / 541-Black-and-white / 2124-Spray painting / 2853-Aerosol paint / \n",
"0-Games / 15-Cartoon / 759-Icing / \n",
"20-Trailer / \n",
"0-Games / 8-Football / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 161-Train station / 224-Rapid transit / \n",
"6-Animation / 15-Cartoon / 183-Manga / \n",
"0-Games / 2-Video game / 33-Weapon / 34-Action-adventure game / 380-Assassin's Creed / 1048-Assassin's Creed / \n",
"66-Bollywood / \n",
"9-Music video / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 25-Toy / 41-Sports car / 92-Radio-controlled model / 135-Drifting / 202-Radio-controlled car / 569-Lamborghini / 926-Traxxas / 4631-Lamborghini Diablo / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"3-Concert / 7-Musician / \n",
"1-Vehicle / 147-River / 794-Bridge / \n",
"2641-Radha / \n",
"1-Vehicle / 27-Motorcycle / 158-Tire / 806-Chopper / \n",
"5-Dance / 16-Performance art / 49-School / 95-Talent show / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 81-Athlete / \n",
"20-Trailer / \n",
"0-Games / 2-Video game / 55-Video game console / 199-Wii / 320-Super Smash Bros. / 335-Nintendo 3DS / 443-Wii U / 697-Super Smash Bros. for Nintendo 3DS and Wii U / \n",
"164-Beach / 248-Hotel / 369-Resort / \n",
"49-School / 949-Rowing / \n",
"110-Album / \n",
"22-Nature / 490-Rain / 958-Thunderstorm / \n",
"3-Concert / \n",
"22-Nature / 423-Coast / \n",
"419-Earth / 593-Moon / 1686-Orbit / \n",
"0-Games / 1976-Nostale / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"9-Music video / 2461-DatPiff / \n",
"85-Combat / 209-University / \n",
"0-Games / 1-Vehicle / 2-Video game / 11-Motorsport / 17-Racing / 55-Video game console / 57-Race track / 199-Wii / 456-Go-kart / 518-Mario Kart / 3346-Super Mario Kart / \n",
"0-Games / 8-Football / 90-Sports game / 180-FIFA 15 / 480-FIFA 13 / \n",
"0-Games / 2-Video game / 55-Video game console / 199-Wii / \n",
"3-Concert / \n",
"1-Vehicle / 74-Truck / 259-Fire / 371-Firefighter / 796-Fire engine / 2414-Firefighting / \n",
"9-Music video / \n",
"36-Piano / 53-Keyboard / \n",
"23-Mobile phone / 29-Smartphone / 487-Samsung Electronics / \n",
"1-Vehicle / 4-Car / 449-Honda / 552-Honda Civic / \n",
"6-Animation / 15-Cartoon / 3600-The Jungle Book / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 127-Xbox 360 / 194-Call of Duty: Modern Warfare 2 / 302-Call of Duty 4: Modern Warfare / 464-First-person Shooter / \n",
"31-Disc jockey / \n",
"5-Dance / 49-School / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 1124-Yamaha YZF-R1 / \n",
"3-Concert / 5-Dance / 350-Circus / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"0-Games / 2-Video game / 159-World of Warcraft / 1356-Paladin / \n",
"7-Musician / 14-Guitar / 99-Christmas / 2252-Steel guitar / 2809-Pedal steel guitar / \n",
"3-Concert / 9-Music video / \n",
"144-Amusement park / 321-Roller coaster / 334-Amusement ride / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 30-Drummer / \n",
"0-Games / 59-Winter sport / 112-Ice skating / 142-Hockey / 312-Ice rink / \n",
"118-Grand Theft Auto V / 120-Winter / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 24-String instrument / 38-Orchestra / 150-Violin / 401-Cello / 450-Viola / 545-Quartet (ensemble) / 1002-String quartet / \n",
"6-Animation / 20-Trailer / 298-Television advertisement / \n",
"10-Animal / 22-Nature / 80-Horse / 290-Pony / 956-Glasses / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / \n",
"359-Harry Potter (Literary Series) / 4489-Harry Potter and the Philosopher's Stone / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 135-Drifting / \n",
"7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"85-Combat / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 404-Livestock / 438-Mare / 523-Cattle / 1586-Cowboy / \n",
"1-Vehicle / 4-Car / 18-Outdoor recreation / 167-Four-wheel drive / 198-Off-road vehicle / 277-Sport utility vehicle / 360-Jeep / \n",
"1-Vehicle / 4-Car / 50-Aircraft / 114-Engine / 129-Aviation / 151-Landing / 451-Cockpit / \n",
"463-Cricket / \n",
"14-Guitar / \n",
"0-Games / 2-Video game / 241-Fighting game / 764-The King of Fighters / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / 197-Trail / 213-Forest / 220-Mountain bike / 256-Tennis / 491-Enduro / \n",
"0-Games / 2-Video game / 55-Video game console / 125-Television / 413-PlayStation 2 / 581-PlayStation / 1349-USB flash drive / \n",
"573-Paragliding / 765-Parachute / \n",
"7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / 639-Fiddle / 670-Banjo / \n",
"0-Games / 8-Football / 90-Sports game / 180-FIFA 15 / \n",
"0-Games / 2-Video game / 102-Building / 1027-Terraria / \n",
"33-Weapon / 121-Photography / 3258-Leica Camera / 4195-Binoculars / \n",
"0-Games / 2-Video game / 75-Wrestling / 85-Combat / 90-Sports game / 3764-EA Sports UFC / \n",
"49-School / 209-University / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 76-Ball / \n",
"9-Music video / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 1448-Vespa / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 18-Outdoor recreation / 27-Motorcycle / 105-Motorcycling / 172-Motocross / 197-Trail / 213-Forest / 220-Mountain bike / 491-Enduro / \n",
"3-Concert / 5-Dance / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / \n",
"3-Concert / 5-Dance / \n",
"1345-Dean Winchester / \n",
"0-Games / 2-Video game / 35-Minecraft / 55-Video game console / 111-PlayStation 3 / 127-Xbox 360 / 181-Xbox / \n",
"2062-Kidney / \n",
"23-Mobile phone / \n",
"200-Doll / 4237-Ball-jointed doll / \n",
"6-Animation / 15-Cartoon / 61-Art / 97-Drawing / 148-Painting / 290-Pony / \n",
"7-Musician / 13-Musical ensemble / \n",
"5-Dance / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 212-Highway / 286-Bus / \n",
"15-Cartoon / \n",
"0-Games / 8-Football / \n",
"20-Trailer / \n",
"5-Dance / \n",
"14-Guitar / \n",
"2-Video game / 192-Battlefield / 374-Battlefield 3 / \n",
"12-Food / 295-Chocolate / 442-Bread / \n",
"1-Vehicle / 4-Car / 360-Jeep / 395-Camping / 707-Roof / 914-Tent / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 380-Assassin's Creed / 1522-Assassin's Creed II / \n",
"1-Vehicle / 4-Car / 41-Sports car / \n",
"0-Games / 35-Minecraft / 3175-Hide-and-seek / \n",
"3-Concert / 5-Dance / 16-Performance art / 31-Disc jockey / 3458-DMC World DJ Championships / \n",
"142-Hockey / 288-Underwater / \n",
"992-Rocket / 1508-Spacecraft / 1779-Satellite / 4397-Rocket launch / \n",
"1-Vehicle / 50-Aircraft / 3752-Airship / \n",
"12-Food / 367-Drink / 904-Cocktail / 1669-Vodka / \n",
"293-Slide show / \n",
"7-Musician / \n",
"2-Video game / 33-Weapon / 474-Destiny / 1130-Armour / \n",
"75-Wrestling / 85-Combat / 122-Weight training / 204-Gym / 1584-Kettlebell / \n",
"420-The Elder Scrolls V: Skyrim / 441-The Elder Scrolls / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 1200-Ultralight aviation / \n",
"1-Vehicle / 8-Football / 286-Bus / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 135-Drifting / 2718-Gran Turismo 4 / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 301-Meal / 442-Bread / 966-Sandwich / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / 100-Electric guitar / \n",
"0-Games / 2-Video game / 910-Dragon Quest / \n",
"5-Dance / \n",
"5-Dance / \n",
"9-Music video / \n",
"1-Vehicle / 88-Machine / 124-Tractor / 156-Agriculture / \n",
"0-Games / 2-Video game / 412-Monster / \n",
"1088-Infomercial / \n",
"6-Animation / 15-Cartoon / 123-Naruto / 183-Manga / 541-Black-and-white / \n",
"59-Winter sport / 83-Skateboarding / 281-Snowboarding / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 135-Drifting / \n",
"67-Cycling / 69-Bicycle / 221-Skatepark / 377-BMX bike / \n",
"9-Music video / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"0-Games / 51-Strategy video game / 373-Clash of Clans / 924-Trophy / \n",
"6-Animation / 10-Animal / 15-Cartoon / 1428-Rooster / 2832-Looney Tunes / \n",
"14-Guitar / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"209-University / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 124-Tractor / 252-Tractor pulling / \n",
"0-Games / 19-PC game / 33-Weapon / 596-Monster Hunter / 3910-Mackerel / \n",
"3-Concert / 7-Musician / 14-Guitar / 16-Performance art / \n",
"28-Fashion / 56-Hair / 106-Hairstyle / 340-Afro-textured hair / \n",
"78-Wedding / 293-Slide show / 590-Wedding dress / \n",
"1-Vehicle / 40-Road / 64-Transport / 74-Truck / \n",
"6-Animation / 1866-Katara / \n",
"402-Jewellery / 1082-Knot / 2507-Macramé / \n",
"9-Music video / \n",
"125-Television / \n",
"5-Dance / 130-Gymnastics / 892-Aerobics / 2228-Sport aerobics / \n",
"3-Concert / 7-Musician / 16-Performance art / 77-Arena / \n",
"6-Animation / 15-Cartoon / 755-Hetalia: Axis Powers / \n",
"3-Concert / 1143-Marimba / 1734-Xylophone / 2878-Vibraphone / \n",
"433-Construction / 976-Demolition / \n",
"3-Concert / 1680-Seafight / \n",
"99-Christmas / 415-Sewing / 476-Textile / 964-Christmas tree / 997-Sewing machine / 2329-Felt / 2499-Christmas ornament / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / 296-Saxophone / \n",
"144-Amusement park / 321-Roller coaster / 334-Amusement ride / \n",
"359-Harry Potter (Literary Series) / 3719-Harry Potter / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 94-Dashcam / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 235-Lipstick / 291-Rouge / 436-Concealer / 1012-Lip gloss / \n",
"0-Games / 1-Vehicle / 2-Video game / 199-Wii / 443-Wii U / 456-Go-kart / 518-Mario Kart / 685-Luigi / 1831-Mario Kart 8 / \n",
"162-Medicine / \n",
"2-Video game / 6-Animation / 506-Garry's Mod / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 51-Strategy video game / 125-Television / 165-League of Legends / \n",
"779-Point Blank / \n",
"298-Television advertisement / 337-Advertising / \n",
"694-Tai chi / \n",
"85-Combat / 1623-Shaolin Kung Fu / \n",
"3-Concert / 7-Musician / 3208-Lowlands / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 76-Ball / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 105-Motorcycling / 172-Motocross / 491-Enduro / \n",
"1-Vehicle / 18-Outdoor recreation / 42-Fishing / 87-Boat / 225-Lake / 753-Sailboat / 917-Sail / \n",
"12-Food / 26-Cooking / 32-Recipe / 109-Cooking show / 226-Dessert / 232-Cake / 295-Chocolate / 985-Strawberry / 1208-Coconut / \n",
"0-Games / 1138-Tower of Saviors / \n",
"42-Fishing / \n",
"7-Musician / 13-Musical ensemble / 44-Drums / 203-Marching band / 230-Brass instrument / 250-Parade / \n",
"110-Album / 754-Gramophone record / \n",
"47-Personal computer / 141-Microsoft Windows / 145-Tablet computer / 228-Laptop / 513-Computer keyboard / 631-Cube / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 79-American football / \n",
"6-Animation / 15-Cartoon / 61-Art / 97-Drawing / 396-One Piece / 408-Sketch / \n",
"1873-Crossbow / \n",
"6-Animation / 15-Cartoon / \n",
"170-Home improvement / 574-Elevator / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 760-iPhone 5s / 1774-iPhone 5c / \n",
"10-Animal / 25-Toy / 48-Pet / 132-Bird / 598-Parrot / \n",
"3-Concert / \n",
"31-Disc jockey / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 210-Cookware and bakeware / 1186-Seafood / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / \n",
"2-Video game / 176-PlayStation 4 / 223-Weather / 441-The Elder Scrolls / 1188-Uncharted / 1401-The Witcher 3: Wild Hunt / 1919-The Elder Scrolls Online / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 270-Sedan / 901-Mitsubishi / 1122-Mitsubishi Lancer Evolution / 1413-Mitsubishi Lancer / \n",
"22-Nature / 164-Beach / 233-Ocean / 423-Coast / \n",
"410-Teacher / \n",
"25-Toy / 140-LEGO / 264-Star Wars / 852-Lego Star Wars / 884-Lego minifigure / \n",
"31-Disc jockey / \n",
"0-Games / 51-Strategy video game / \n",
"9-Music video / \n",
"20-Trailer / 121-Photography / 633-Photograph / \n",
"4591-Texas Longhorns / \n",
"10-Animal / 18-Outdoor recreation / 217-Hunting / 636-Rabbit / 2135-Bowhunting / \n",
"1-Vehicle / 18-Outdoor recreation / 22-Nature / 67-Cycling / 69-Bicycle / 197-Trail / 220-Mountain bike / 284-Mountain biking / \n",
"78-Wedding / 121-Photography / 190-Bride / 293-Slide show / 590-Wedding dress / 2269-Wedding photography / \n",
"5-Dance / 13-Musical ensemble / \n",
"9-Music video / \n",
"3-Concert / \n",
"9-Music video / \n",
"49-School / 410-Teacher / 829-Bead / \n",
"12-Food / 232-Cake / 607-Lace / 759-Icing / 1102-Cake decorating / 1365-Fondant icing / 1541-Buttercream / 2255-Stencil / 2941-Wedding cake / \n",
"10-Animal / 22-Nature / 215-Eating / 540-Insect / 642-Leaf / 1733-Ant / \n",
"18-Outdoor recreation / 287-Human swimming / 949-Rowing / 2542-Cross / \n",
"1-Vehicle / 4-Car / \n",
"59-Winter sport / 112-Ice skating / 142-Hockey / 4454-Hockey stick / \n",
"28-Fashion / 45-Cosmetics / 1572-Husband / \n",
"0-Games / 2-Video game / 55-Video game console / 111-PlayStation 3 / 3605-Puppetry / \n",
"75-Wrestling / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / \n",
"102-Building / \n",
"0-Games / 8-Football / 65-Kick / 76-Ball / 81-Athlete / 1037-Grass / \n",
"3-Concert / \n",
"465-Microphone / 1954-Power supply / \n",
"6-Animation / 15-Cartoon / \n",
"322-Mixtape / \n",
"20-Trailer / \n",
"12-Food / 348-Fruit / 2000-Mango / \n",
"8-Football / 90-Sports game / 180-FIFA 15 / \n",
"1-Vehicle / 22-Nature / 27-Motorcycle / 40-Road / 245-BMW / 2763-BMW R1200GS / \n",
"18-Outdoor recreation / 21-Stadium / 448-Planet / \n",
"0-Games / 2-Video game / 432-Kingdom Hearts / 1908-Kingdom Hearts Birth by Sleep / 2934-Zack Fair / \n",
"12-Food / 25-Toy / 26-Cooking / 179-Kitchen / 1154-Anpanman / \n",
"10-Animal / 48-Pet / 71-Dog / 881-German Shepherd / \n",
"2693-Puma SE / \n",
"0-Games / 211-RuneScape / \n",
"38-Orchestra / 1358-Canyon / \n",
"1203-Backpacking / \n",
"1-Vehicle / 4-Car / 574-Elevator / 1391-Kia / \n",
"2-Video game / 47-Personal computer / 131-Computer / 309-Computer hardware / 602-Central processing unit / 621-Asus / 1045-Computer case / \n",
"22-Nature / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 297-Grand Theft Auto IV / 658-Grand Theft Auto: The Lost and Damned / \n",
"110-Album / \n",
"12-Food / 1550-Newspaper / \n",
"205-Newscaster / 1920-Copper / \n",
"0-Games / 2-Video game / 33-Weapon / 474-Destiny / 1578-Red Dead Redemption / \n",
"0-Games / 2-Video game / 774-Galaxy / 1397-Super Mario Galaxy / \n",
"25-Toy / 435-Action figure / 2120-Tears / \n",
"5-Dance / 28-Fashion / \n",
"0-Games / 6-Animation / \n",
"6-Animation / 15-Cartoon / \n",
"3540-Violet / \n",
"10-Animal / 48-Pet / 71-Dog / 1252-Dog agility / 2083-Border Collie / 2087-Collie / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"113-House / 136-Gardening / 170-Home improvement / \n",
"9-Music video / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / \n",
"122-Weight training / \n",
"3-Concert / 7-Musician / \n",
"66-Bollywood / \n",
"439-Barbie / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / \n",
"31-Disc jockey / \n",
"9-Music video / \n",
"12-Food / 26-Cooking / 32-Recipe / \n",
"5-Dance / 16-Performance art / \n",
"9-Music video / \n",
"402-Jewellery / \n",
"9-Music video / \n",
"208-Wood / 259-Fire / 1739-Fireplace / \n",
"20-Trailer / \n",
"0-Games / 6-Animation / 88-Machine / 240-Arcade game / 535-Slot machine / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 1434-Saddle / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 500-Rice / 1227-Sushi / 1569-Steamed rice / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"3861-Toner / \n",
"0-Games / 2-Video game / 3366-Amnesia: The Dark Descent / \n",
"67-Cycling / 69-Bicycle / 220-Mountain bike / 520-Bicycle frame / 645-Road bicycle / \n",
"46-Choir / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"10-Animal / 91-Fish / 258-Aquarium / 1378-Multiplayer online battle arena / \n",
"28-Fashion / 49-School / 56-Hair / 106-Hairstyle / 405-Braid / 1064-French braid / \n",
"1-Vehicle / 42-Fishing / 87-Boat / \n",
"20-Trailer / 1551-The Phantom of the Opera / \n",
"0-Games / 54-Highlight film / \n",
"3-Concert / 5-Dance / \n",
"6-Animation / 20-Trailer / 126-The Walt Disney Company / \n",
"56-Hair / 106-Hairstyle / 1271-Beauty salon / \n",
"3-Concert / 7-Musician / 36-Piano / 104-Pianist / 150-Violin / \n",
"116-Pokémon / 186-Pokémon / 372-Playing card / 690-Pokémon X and Y / 1021-Booster pack / 1049-Collectible card game / 1062-Pokémon Trading Card Game / 2911-Charizard / \n",
"23-Mobile phone / 29-Smartphone / 141-Microsoft Windows / \n",
"0-Games / 2-Video game / 535-Slot machine / \n",
"31-Disc jockey / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 92-Radio-controlled model / 167-Four-wheel drive / 202-Radio-controlled car / \n",
"6-Animation / 15-Cartoon / \n",
"75-Wrestling / 85-Combat / \n",
"36-Piano / 1023-Rose / \n",
"372-Playing card / 746-Card manipulation / \n",
"117-Boxing / \n",
"2-Video game / 85-Combat / 89-Comics / 294-Batman / 2146-Bane / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 2626-Shih Tzu / \n",
"7-Musician / 13-Musical ensemble / \n",
"1-Vehicle / 4-Car / 88-Machine / 207-Exhaust system / 1661-Volkswagen Gol / \n",
"20-Trailer / 748-Iron Man / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 38-Orchestra / 39-Drums / 44-Drums / \n",
"460-Yu-Gi-Oh! Trading Card Game / \n",
"3-Concert / 30-Drummer / 39-Drums / \n",
"18-Outdoor recreation / 22-Nature / \n",
"14-Guitar / 100-Electric guitar / \n",
"31-Disc jockey / \n",
"20-Trailer / \n",
"0-Games / 1-Vehicle / 2-Video game / 3252-Burnout Paradise / \n",
"0-Games / 2-Video game / 264-Star Wars / 714-Jedi / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 138-Call of Duty: Black Ops II / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / 716-Coral / \n",
"1-Vehicle / 4-Car / 154-Wheel / 493-Rim / \n",
"1-Vehicle / 62-Train / 479-Graffiti / \n",
"3-Concert / 68-Lighting / \n",
"123-Naruto / 272-iPod / \n",
"31-Disc jockey / \n",
"242-Paper / 415-Sewing / 468-Biology / 1525-Wallet / 2950-Zipper / \n",
"1-Vehicle / \n",
"0-Games / 2-Video game / 19-PC game / 1091-Saints Row / 2695-Saints Row IV / \n",
"3-Concert / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 30-Drummer / \n",
"0-Games / 35-Minecraft / \n",
"2625-Gabrielle / \n",
"10-Animal / 91-Fish / \n",
"0-Games / 2-Video game / 85-Combat / 317-PlayStation / 511-Mortal Kombat / \n",
"28-Fashion / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"6-Animation / 801-Inuyasha / \n",
"1-Vehicle / 12-Food / 124-Tractor / 156-Agriculture / 174-Farm / 249-Heavy equipment / 660-Rural area / 984-Silage / 1473-Forage harvester / \n",
"122-Weight training / \n",
"376-Flute / \n",
"2454-Notebook / \n",
"0-Games / 35-Minecraft / \n",
"48-Pet / 2087-Collie / \n",
"1-Vehicle / 4-Car / 349-Chevrolet / 406-Chevrolet / 3190-Chevrolet Cruze / \n",
"6-Animation / 15-Cartoon / 183-Manga / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 230-Brass instrument / \n",
"955-Police officer / \n",
"240-Arcade game / 781-Pachinko / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 178-Samsung Galaxy / 3648-Samsung Galaxy S III Mini / \n",
"9-Music video / \n",
"0-Games / 734-DayZ / 1216-DayZ / \n",
"61-Art / 242-Paper / 370-Origami / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"15-Cartoon / 286-Bus / \n",
"61-Art / 2856-Decoupage / \n",
"88-Machine / 208-Wood / 261-Tool / 400-Woodturning / 972-Lathe / 1384-Greenhouse / \n",
"1-Vehicle / 11-Motorsport / 2875-Quadracycle / \n",
"6-Animation / 15-Cartoon / 251-Sonic the Hedgehog / \n",
"3-Concert / 5-Dance / 133-Nightclub / 718-Duck / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 1400-rFactor / \n",
"3-Concert / 5-Dance / 30-Drummer / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 301-Meal / 542-Potato / 557-Salad / 1024-Culinary art / \n",
"1-Vehicle / 4-Car / 245-BMW / 1709-BMW 3 Series (E46) / 2736-BMW 5 Series (E39) / \n",
"0-Games / 2-Video game / 43-Call of Duty / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 398-The Legend of Zelda / 692-Link / 2423-The Legend of Zelda: Skyward Sword / \n",
"1-Vehicle / 4-Car / 74-Truck / 154-Wheel / 158-Tire / 3601-Fender / \n",
"12-Food / 1085-Barbecue grill / \n",
"10-Animal / 22-Nature / 132-Bird / 1905-Owl / \n",
"20-Trailer / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 169-Rallying / 680-Subaru / 1041-Subaru Impreza / 1286-Subaru / \n",
"1-Vehicle / 23-Mobile phone / 37-Gadget / 47-Personal computer / 145-Tablet computer / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 30-Drummer / \n",
"358-Rugby football / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / 1313-Strum / 2241-Plectrum / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 192-Battlefield / 383-Battlefield 4 / 1267-Battlefield Hardline / \n",
"1-Vehicle / 4-Car / 552-Honda Civic / \n",
"3-Concert / 7-Musician / 14-Guitar / 16-Performance art / \n",
"1-Vehicle / 4-Car / 806-Chopper / \n",
"3-Concert / 7-Musician / \n",
"625-Factory / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 651-Stock car racing / 728-Dirt track racing / \n",
"0-Games / 8-Football / 90-Sports game / \n",
"3-Concert / \n",
"28-Fashion / 45-Cosmetics / 153-Dress / 206-Shoe / 785-Jeans / 793-Shirt / \n",
"1-Vehicle / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 197-Trail / 220-Mountain bike / 278-GoPro / 284-Mountain biking / \n",
"23-Mobile phone / 29-Smartphone / 101-Telephone / \n",
"110-Album / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 70-Driving / \n",
"9-Music video / 260-Prayer / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / 224-Rapid transit / \n",
"0-Games / 240-Arcade game / 241-Fighting game / 764-The King of Fighters / \n",
"163-Running / 206-Shoe / 368-Sneakers / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 124-Tractor / 252-Tractor pulling / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 55-Video game console / 973-Nintendo 64 / 1343-Super Robot Wars / \n",
"1030-Trampoline / \n",
"20-Trailer / \n",
"20-Trailer / \n",
"10-Animal / 48-Pet / 132-Bird / 215-Eating / 307-Hand / 598-Parrot / 1341-Parakeet / \n",
"31-Disc jockey / 861-Mixing console / 3024-Numark Industries / \n",
"6-Animation / 15-Cartoon / 89-Comics / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / 437-Cream / 1268-Tails / \n",
"25-Toy / 140-LEGO / 1478-Lego Ninjago / \n",
"3-Concert / 10-Animal / \n",
"0-Games / 2-Video game / 8-Football / 90-Sports game / 180-FIFA 15 / 480-FIFA 13 / 848-FIFA 12 / \n",
"9-Music video / 121-Photography / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / \n",
"261-Tool / 315-Knife / 2017-Sharpening / 3880-Knife sharpening / \n",
"347-Climbing / 828-Rock climbing / \n",
"6-Animation / 89-Comics / 396-One Piece / \n",
"130-Gymnastics / \n",
"12-Food / 367-Drink / 1065-Bartender / \n",
"2-Video game / 445-Guitar Hero / \n",
"36-Piano / 53-Keyboard / 104-Pianist / \n",
"0-Games / 2-Video game / 20-Trailer / 686-Diablo III / \n",
"2-Video game / 20-Trailer / \n",
"81-Athlete / \n",
"6-Animation / 15-Cartoon / 185-Dragon Ball / 244-Goku / \n",
"45-Cosmetics / 385-Face / \n",
"8-Football / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / \n",
"56-Hair / 106-Hairstyle / \n",
"6-Animation / 4221-Saiyuki / \n",
"30-Drummer / 39-Drums / 44-Drums / \n",
"159-World of Warcraft / 171-Warcraft / 994-Cue stick / \n",
"3-Concert / \n",
"121-Photography / 155-Camera / \n",
"9-Music video / \n",
"10-Animal / 22-Nature / 86-Plant / 136-Gardening / 540-Insect / 775-Bee / 784-Honey / 1121-Beehive / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 241-Fighting game / 1638-Soulcalibur / 2633-Soulcalibur V / \n",
"10-Animal / 12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / 91-Fish / 893-Fish as food / \n",
"0-Games / 2-Video game / 6-Animation / 34-Action-adventure game / 858-Carl Johnson / \n",
"9-Music video / \n",
"20-Trailer / 66-Bollywood / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 114-Engine / \n",
"1-Vehicle / 4-Car / 270-Sedan / 515-Dodge / \n",
"5-Dance / 49-School / \n",
"18-Outdoor recreation / 42-Fishing / 902-Fishing bait / 1306-Carp fishing / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / 224-Rapid transit / \n",
"238-Nail / 300-Nail art / 304-Nail polish / 307-Hand / 324-Finger / 338-Manicure / 1325-Gel / \n",
"5-Dance / \n",
"458-Gold / 912-Silver / \n",
"20-Trailer / \n",
"0-Games / 6-Animation / 1934-Hunter × Hunter / \n",
"15-Cartoon / 294-Batman / \n",
"37-Gadget / 47-Personal computer / 228-Laptop / 900-MacBook / 1086-MacBook Pro / 1657-MacBook Air / \n",
"0-Games / 35-Minecraft / \n",
"66-Bollywood / 260-Prayer / 1019-Christ / \n",
"67-Cycling / 69-Bicycle / \n",
"12-Food / \n",
"31-Disc jockey / 44-Drums / \n",
"1-Vehicle / 74-Truck / 259-Fire / 371-Firefighter / 796-Fire engine / 1020-Emergency vehicle / \n",
"121-Photography / \n",
"0-Games / 51-Strategy video game / 373-Clash of Clans / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 101-Telephone / \n",
"8-Football / 54-Highlight film / 79-American football / \n",
"9-Music video / \n",
"9-Music video / \n",
"5-Dance / 16-Performance art / 95-Talent show / 115-Ballet / \n",
"14-Guitar / \n",
"66-Bollywood / \n",
"3-Concert / \n",
"47-Personal computer / 309-Computer hardware / 700-Hard disk drive / \n",
"0-Games / 2-Video game / 89-Comics / 185-Dragon Ball / 244-Goku / 1304-Cell / 2144-Dragon Ball: Raging Blast 2 / 2384-Dragon Ball: Raging Blast / \n",
"8-Football / 54-Highlight film / \n",
"59-Winter sport / 112-Ice skating / 142-Hockey / \n",
"7-Musician / 24-String instrument / 36-Piano / 150-Violin / \n",
"102-Building / 113-House / 170-Home improvement / 276-Room / 353-Wall / 580-Bedroom / \n",
"75-Wrestling / 85-Combat / \n",
"219-Accordion / \n",
"8-Football / 76-Ball / \n",
"322-Mixtape / \n",
"6-Animation / 749-Figurine / \n",
"3-Concert / \n",
"3-Concert / 16-Performance art / \n",
"1-Vehicle / 4-Car / \n",
"86-Plant / 136-Gardening / 137-Tree / 279-Garden / \n",
"3-Concert / 16-Performance art / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 96-Soldier / 194-Call of Duty: Modern Warfare 2 / 302-Call of Duty 4: Modern Warfare / \n",
"12-Food / 18-Outdoor recreation / 86-Plant / 124-Tractor / 156-Agriculture / 459-Plough / 2039-Soybean / \n",
"0-Games / 2-Video game / 1091-Saints Row / 1695-Saints Row: The Third / \n",
"14-Guitar / \n",
"170-Home improvement / 179-Kitchen / 488-Door / 536-Glass / 1272-Cabinetry / \n",
"246-Furniture / 462-Table / 888-Living room / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 2517-Chevrolet Opala / \n",
"0-Games / 1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 70-Driving / 84-Snow / 120-Winter / 135-Drifting / 245-BMW / 874-BMW M3 / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / \n",
"2531-Dojo / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"1-Vehicle / 342-Windows Media Video / 2194-Wind power / \n",
"78-Wedding / 190-Bride / 293-Slide show / \n",
"2-Video game / 111-PlayStation 3 / \n",
"47-Personal computer / 72-iPhone / 131-Computer / 145-Tablet computer / 189-iPad / 228-Laptop / 513-Computer keyboard / 1150-iPad 2 / \n",
"3-Concert / \n",
"121-Photography / 155-Camera / 633-Photograph / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 40-Road / 2788-Volkswagen Golf Mk2 / 3762-PSA HDi engine / 3909-Hyundai Elantra / \n",
"6-Animation / 9-Music video / 25-Toy / 200-Doll / 412-Monster / 503-Monster High / \n",
"5-Dance / 16-Performance art / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 230-Brass instrument / 634-Trombone / \n",
"0-Games / 2-Video game / 19-PC game / 192-Battlefield / 383-Battlefield 4 / 3005-MicroVolts / \n",
"5-Dance / 16-Performance art / 126-The Walt Disney Company / \n",
"0-Games / 1-Vehicle / 2-Video game / 62-Train / \n",
"66-Bollywood / \n",
"0-Games / 2-Video game / 90-Sports game / 561-Madden NFL / 1899-E-book / \n",
"1125-God of War / 2754-God of War II / \n",
"122-Weight training / 214-Muscle / 1250-Biceps / \n",
"1283-Pancake / \n",
"31-Disc jockey / \n",
"22-Nature / 86-Plant / 156-Agriculture / 217-Hunting / 1229-Magnet / \n",
"1-Vehicle / 4-Car / 34-Action-adventure game / 118-Grand Theft Auto V / 299-Volkswagen Passenger Cars / 659-Van / 2486-Scooby-Doo / \n",
"18-Outdoor recreation / 59-Winter sport / 112-Ice skating / 316-Figure skating / 632-Ice dancing / \n",
"3-Concert / 31-Disc jockey / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 105-Motorcycling / 172-Motocross / \n",
"1-Vehicle / 62-Train / \n",
"2176-Fist of the North Star / \n",
"0-Games / 6-Animation / 15-Cartoon / 2412-Fate/stay night / \n",
"14-Guitar / 100-Electric guitar / \n",
"1-Vehicle / 4-Car / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / \n",
"6-Animation / 10-Animal / 15-Cartoon / \n",
"211-RuneScape / 366-Dragon / \n",
"429-Website / \n",
"1-Vehicle / 4-Car / 5-Dance / \n",
"12-Food / 612-Beer / \n",
"7-Musician / 14-Guitar / 63-Acoustic guitar / 99-Christmas / \n",
"60-Basketball / \n",
"2-Video game / \n",
"0-Games / 2-Video game / 37-Gadget / 55-Video game console / 362-PlayStation Portable / 581-PlayStation / \n",
"141-Microsoft Windows / \n",
"1-Vehicle / 62-Train / 507-Thomas the Tank Engine / 837-Roblox / \n",
"31-Disc jockey / \n",
"999-Elephant / \n",
"0-Games / 8-Football / \n",
"20-Trailer / \n",
"9-Music video / 122-Weight training / 204-Gym / 214-Muscle / \n",
"402-Jewellery / 907-Necklace / 3012-Jilbāb / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"6-Animation / 123-Naruto / \n",
"6-Animation / 9-Music video / 15-Cartoon / \n",
"0-Games / 60-Basketball / 77-Arena / 81-Athlete / 168-Basketball moves / 255-Slam dunk / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"5-Dance / 9-Music video / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 1599-Bone / 1723-Chihuahua / 2311-Human tooth / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 30-Drummer / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 352-Need for Speed / 875-Need for Speed: Most Wanted / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / \n",
"0-Games / 59-Winter sport / 81-Athlete / 112-Ice skating / 142-Hockey / 312-Ice rink / 4285-The Sports Network / \n",
"911-Studio / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 1040-Street racing / \n",
"308-Ship / 605-Cruise ship / \n",
"8-Football / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 94-Dashcam / 212-Highway / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"66-Bollywood / \n",
"1-Vehicle / 10-Animal / 50-Aircraft / 132-Bird / 471-Wing / \n",
"8-Football / \n",
"1-Vehicle / 4-Car / 70-Driving / 167-Four-wheel drive / 270-Sedan / 277-Sport utility vehicle / 901-Mitsubishi / 3896-Mitsubishi Outlander / \n",
"22-Nature / 86-Plant / \n",
"28-Fashion / 45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 291-Rouge / \n",
"99-Christmas / \n",
"31-Disc jockey / \n",
"14-Guitar / 529-Squat / \n",
"1-Vehicle / 4-Car / 2690-Škoda Octavia / \n",
"56-Hair / 954-Dreadlocks / \n",
"10-Animal / 22-Nature / 636-Rabbit / 1047-Easter egg / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 344-Coupé / 1074-Hyundai / 3316-Hyundai Genesis / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 651-Stock car racing / 728-Dirt track racing / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / 100-Electric guitar / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"0-Games / 8-Football / 21-Stadium / 77-Arena / \n",
"96-Soldier / 1573-United States Army / \n",
"12-Food / 26-Cooking / 389-Restaurant / \n",
"28-Fashion / 56-Hair / 106-Hairstyle / 340-Afro-textured hair / 405-Braid / 857-Afro / 954-Dreadlocks / 1812-Hair twists / \n",
"130-Gymnastics / \n",
"0-Games / 2-Video game / 6-Animation / 1618-Rayman / 4058-Rayman Origins / \n",
"20-Trailer / 1014-Movieclips / 1756-Predator / 2011-Alien / \n",
"3-Concert / 5-Dance / 253-Carnival / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 247-Stallion / 289-Dressage / 351-Jumping / \n",
"695-Balloon / 1972-Toy balloon / \n",
"56-Hair / \n",
"863-Body piercing / 2118-Tongue / 4123-Tongue piercing / \n",
"0-Games / 35-Minecraft / 395-Camping / \n",
"7-Musician / 13-Musical ensemble / 24-String instrument / \n",
"20-Trailer / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"144-Amusement park / 334-Amusement ride / 3576-Iguana / \n",
"31-Disc jockey / 342-Windows Media Video / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 30-Drummer / \n",
"296-Saxophone / \n",
"45-Cosmetics / 193-Eye liner / 218-Eyelash / \n",
"4108-Orangutan / \n",
"310-The Sims / 564-The Sims 3 / \n",
"5-Dance / \n",
"1-Vehicle / 4-Car / 34-Action-adventure game / 41-Sports car / 201-Grand Theft Auto: San Andreas / 297-Grand Theft Auto IV / \n",
"66-Bollywood / \n",
"102-Building / 433-Construction / \n",
"6-Animation / 15-Cartoon / 116-Pokémon / \n",
"2-Video game / 20-Trailer / 34-Action-adventure game / 264-Star Wars / 3999-Star Wars: The Force Unleashed II / \n",
"10-Animal / 2356-Comb / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 194-Call of Duty: Modern Warfare 2 / \n",
"98-Festival / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 74-Truck / 92-Radio-controlled model / 135-Drifting / 167-Four-wheel drive / 202-Radio-controlled car / \n",
"434-Ninja / \n",
"1043-Router / \n",
"455-Unidentified flying object / \n",
"9-Music video / \n",
"1-Vehicle / 62-Train / 286-Bus / \n",
"3-Concert / 16-Performance art / \n",
"298-Television advertisement / 337-Advertising / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 38-Orchestra / 44-Drums / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / \n",
"0-Games / 6-Animation / 372-Playing card / 494-Card game / \n",
"248-Hotel / 916-Spa / \n",
"3-Concert / 7-Musician / 36-Piano / 104-Pianist / 115-Ballet / \n",
"10-Animal / 18-Outdoor recreation / 42-Fishing / 91-Fish / 147-River / 268-Recreational fishing / 478-Fishing rod / 510-Fishing lure / 698-Fisherman / 844-Bass / 896-Largemouth bass / 902-Fishing bait / \n",
"22-Nature / \n",
"12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / 109-Cooking show / 226-Dessert / 232-Cake / 236-Baking / 1088-Infomercial / \n",
"1058-Volcano / \n",
"1-Vehicle / 59-Winter sport / 144-Amusement park / 321-Roller coaster / 1829-Sledding / 4446-Bobsleigh / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 1447-Skylanders / 2321-Skylanders: Spyro's Adventure / \n",
"1-Vehicle / 4-Car / 519-Recreational vehicle / 659-Van / \n",
"6-Animation / 15-Cartoon / 185-Dragon Ball / \n",
"5-Dance / \n",
"8-Football / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / \n",
"1-Vehicle / 18-Outdoor recreation / 50-Aircraft / 82-Airplane / 1200-Ultralight aviation / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / \n",
"22-Nature / \n",
"1-Vehicle / 25-Toy / 50-Aircraft / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 227-Water / 266-Helicopter / 392-Radio-controlled helicopter / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 70-Driving / 449-Honda / 552-Honda Civic / \n",
"14-Guitar / \n",
"567-Compact disc / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 47-Personal computer / 72-iPhone / 145-Tablet computer / 189-iPad / 1504-iPad 3 / \n",
"1-Vehicle / 27-Motorcycle / \n",
"313-Book / \n",
"3-Concert / \n",
"0-Games / 10-Animal / 80-Horse / \n",
"59-Winter sport / 84-Snow / 281-Snowboarding / 501-Snowboard / \n",
"31-Disc jockey / \n",
"47-Personal computer / 131-Computer / 1045-Computer case / 1954-Power supply / \n",
"12-Food / 1363-Candle / 3170-Bath & Body Works / \n",
"5-Dance / \n",
"148-Painting / 408-Sketch / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 802-Google Nexus / 2264-Nexus 7 / \n",
"23-Mobile phone / 29-Smartphone / \n",
"971-Alphabet / \n",
"99-Christmas / 1784-Goblin / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 40-Road / 57-Race track / 105-Motorcycling / 718-Duck / 2133-Road racing / \n",
"165-League of Legends / \n",
"6-Animation / 15-Cartoon / 123-Naruto / 2462-Neji Hyuga / 4333-Kiba Inuzuka / \n",
"8-Football / \n",
"12-Food / 227-Water / 238-Nail / 300-Nail art / \n",
"0-Games / 6-Animation / \n",
"121-Photography / 155-Camera / 566-Digital camera / 1117-Camcorder / \n",
"5-Dance / 16-Performance art / 361-Flamenco / \n",
"10-Animal / 48-Pet / 166-Cat / 346-Kitten / \n",
"1-Vehicle / 18-Outdoor recreation / 42-Fishing / 87-Boat / 481-Kayak / \n",
"3-Concert / 2795-Root / \n",
"476-Textile / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 161-Train station / 224-Rapid transit / 854-Diesel engine / \n",
"10-Animal / 91-Fish / 227-Water / 258-Aquarium / 2090-Siamese fighting fish / 3018-Betta / \n",
"12-Food / 26-Cooking / 944-Hamburger / \n",
"67-Cycling / 69-Bicycle / \n",
"15-Cartoon / 1930-Rurouni Kenshin / \n",
"5-Dance / 16-Performance art / \n",
"94-Dashcam / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 38-Orchestra / \n",
"0-Games / 54-Highlight film / 60-Basketball / 77-Arena / 81-Athlete / 168-Basketball moves / \n",
"2-Video game / 34-Action-adventure game / 118-Grand Theft Auto V / 395-Camping / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 1031-Fisheye lens / \n",
"20-Trailer / \n",
"5-Dance / 98-Festival / \n",
"6-Animation / \n",
"578-Cookie / \n",
"2-Video game / 176-PlayStation 4 / 314-Xbox One / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / 471-Wing / 472-Boeing 737 / \n",
"521-Metal / 731-Coin / 959-Metal detector / \n",
"22-Nature / 42-Fishing / 164-Beach / 233-Ocean / 423-Coast / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"1-Vehicle / 4-Car / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 151-Landing / 254-Runway / 918-Brake / 2296-Joystick / \n",
"0-Games / 1043-Router / \n",
"1-Vehicle / 50-Aircraft / 129-Aviation / 157-Jet aircraft / 175-Airliner / 188-Airline / 388-Fighter aircraft / 758-Air force / \n",
"21-Stadium / 59-Winter sport / 77-Arena / 112-Ice skating / 142-Hockey / \n",
"6-Animation / 10-Animal / 15-Cartoon / 20-Trailer / 71-Dog / 1164-Cartoon Network / \n",
"3-Concert / 68-Lighting / \n",
"9-Music video / \n",
"616-The Bible / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"7-Musician / 14-Guitar / 100-Electric guitar / \n",
"88-Machine / 208-Wood / \n",
"0-Games / 2-Video game / 19-PC game / 34-Action-adventure game / 47-Personal computer / 118-Grand Theft Auto V / \n",
"8-Football / \n",
"12-Food / 86-Plant / 279-Garden / 671-Tomato / \n",
"28-Fashion / 45-Cosmetics / 56-Hair / 106-Hairstyle / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"22-Nature / 86-Plant / \n",
"0-Games / 2-Video game / 20-Trailer / 34-Action-adventure game / 656-Metal Gear / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 18-Outdoor recreation / 27-Motorcycle / 172-Motocross / 2014-Vans / \n",
"3-Concert / \n",
"14-Guitar / \n",
"2-Video game / 34-Action-adventure game / 1275-Watch Dogs / \n",
"5-Dance / \n",
"45-Cosmetics / 235-Lipstick / \n",
"6-Animation / 15-Cartoon / 2098-Tom and Jerry / 2112-Jerry Mouse / \n",
"293-Slide show / \n",
"0-Games / 2-Video game / 19-PC game / 257-Counter-Strike / \n",
"23-Mobile phone / 47-Personal computer / 72-iPhone / 145-Tablet computer / 228-Laptop / 700-Hard disk drive / \n",
"1-Vehicle / 4-Car / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"1-Vehicle / 4-Car / 64-Transport / 234-Ford / \n",
"3-Concert / \n",
"1-Vehicle / 27-Motorcycle / 1575-Clutch / \n",
"5-Dance / 16-Performance art / \n",
"5-Dance / 16-Performance art / \n",
"3-Concert / 5-Dance / \n",
"14-Guitar / 24-String instrument / \n",
"5-Dance / 16-Performance art / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 554-iPhone 4 / 675-iPhone 4S / \n",
"0-Games / 1911-StepMania / \n",
"1-Vehicle / 4-Car / 154-Wheel / 158-Tire / 167-Four-wheel drive / 277-Sport utility vehicle / 1688-Suzuki Jimny / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 24-String instrument / 36-Piano / 150-Violin / \n",
"9-Music video / 83-Skateboarding / \n",
"6-Animation / 20-Trailer / \n",
"8-Football / 21-Stadium / \n",
"6-Animation / 15-Cartoon / \n",
"205-Newscaster / \n",
"5-Dance / 325-Vampire / \n",
"15-Cartoon / 126-The Walt Disney Company / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 135-Drifting / \n",
"6-Animation / 20-Trailer / \n",
"9-Music video / \n",
"9-Music video / \n",
"10-Animal / 48-Pet / 71-Dog / \n",
"33-Weapon / 242-Paper / \n",
"981-Sculpture / \n",
"1-Vehicle / 62-Train / 64-Transport / 161-Train station / 224-Rapid transit / \n",
"5-Dance / 39-Drums / 44-Drums / 1825-Cajón / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"14-Guitar / 425-City / \n",
"1-Vehicle / 4-Car / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 161-Train station / \n",
"3-Concert / 7-Musician / 46-Choir / \n",
"12-Food / 466-Gift / \n",
"49-School / 845-Classroom / \n",
"0-Games / 2-Video game / 257-Counter-Strike / 351-Jumping / \n",
"3-Concert / \n",
"0-Games / 6-Animation / \n",
"0-Games / 2-Video game / 62-Train / 229-Halo / 470-Halo 3 / 1161-Halo 2 / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / 422-Call of Duty: Ghosts / \n",
"9-Music video / \n",
"0-Games / 3149-Melty Blood / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 88-Machine / 114-Engine / \n",
"2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / 121-Photography / 155-Camera / 566-Digital camera / 2248-Zoom lens / \n",
"0-Games / 2-Video game / 1166-Dark Souls II / \n",
"1-Vehicle / 3-Concert / 5-Dance / 11-Motorsport / 49-School / 99-Christmas / 135-Drifting / \n",
"28-Fashion / 56-Hair / \n",
"6-Animation / 15-Cartoon / 1826-Higurashi When They Cry / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"12-Food / \n",
"1-Vehicle / 50-Aircraft / 139-Airport / \n",
"551-British Broadcasting Corporation / 2224-Sherlock Holmes / \n",
"54-Highlight film / 79-American football / 81-Athlete / 163-Running / \n",
"10-Animal / 80-Horse / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 70-Driving / 169-Rallying / \n",
"8-Football / \n",
"66-Bollywood / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"12-Food / 353-Wall / 1456-Clothes dryer / 2067-Duct / \n",
"6-Animation / 15-Cartoon / 849-Tales / \n",
"28-Fashion / 45-Cosmetics / 66-Bollywood / 78-Wedding / 153-Dress / 190-Bride / \n",
"97-Drawing / 148-Painting / 1579-Circle / \n",
"2-Video game / 9-Music video / 237-Final Fantasy / 730-Final Fantasy VII / \n",
"3-Concert / 7-Musician / 16-Performance art / 95-Talent show / 99-Christmas / \n",
"20-Trailer / 313-Book / \n",
"0-Games / 51-Strategy video game / 373-Clash of Clans / \n",
"2-Video game / 55-Video game console / 398-The Legend of Zelda / 2659-Universe of The Legend of Zelda / \n",
"67-Cycling / 69-Bicycle / 570-Road bicycle racing / 645-Road bicycle / \n",
"3-Concert / 16-Performance art / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / 99-Christmas / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"18-Outdoor recreation / 42-Fishing / 225-Lake / 268-Recreational fishing / 478-Fishing rod / 510-Fishing lure / 844-Bass / 3037-Micropterus / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 101-Telephone / 178-Samsung Galaxy / 802-Google Nexus / 3089-Nokia N8 / 4059-Nokia N900 / 4128-Nokia N97 / 4424-Nexus One / \n",
"7-Musician / 13-Musical ensemble / 38-Orchestra / 125-Television / \n",
"573-Paragliding / 765-Parachute / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 198-Off-road vehicle / 3386-Hot tub / \n",
"5-Dance / 78-Wedding / \n",
"56-Hair / 871-Hair coloring / 1269-Human hair color / \n",
"2449-Chuck E. Cheese's / \n",
"3-Concert / 16-Performance art / 68-Lighting / \n",
"6-Animation / 3791-Magi: The Labyrinth of Magic / \n",
"10-Animal / 709-Bull / 2091-Destroyer / \n",
"8-Football / 73-Ball / \n",
"46-Choir / 102-Building / 318-Church / 397-Organ / 646-Pipe organ / 870-Chapel / \n",
"2-Video game / 3468-Heroes of Might and Magic / \n",
"3-Concert / \n",
"317-PlayStation / 1196-Dragon Age / 1735-Dragon Age: Inquisition / 2297-Elf / \n",
"93-Comedy / \n",
"56-Hair / \n",
"5-Dance / 16-Performance art / \n",
"42-Fishing / 1008-Kite / \n",
"0-Games / 60-Basketball / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 16-Performance art / 30-Drummer / \n",
"337-Advertising / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / \n",
"0-Games / 2-Video game / 123-Naruto / \n",
"159-World of Warcraft / 171-Warcraft / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"0-Games / 2-Video game / 924-Trophy / 1137-Crash Bandicoot / \n",
"0-Games / 2-Video game / 125-Television / 345-Loudspeaker / 484-Subwoofer / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / 556-Skateboarding trick / \n",
"843-The Doctor / \n",
"61-Art / 479-Graffiti / \n",
"837-Roblox / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"8-Football / 21-Stadium / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"0-Games / 2-Video game / 9-Music video / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / \n",
"3-Concert / 7-Musician / 9-Music video / 13-Musical ensemble / 14-Guitar / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 245-BMW / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"0-Games / 2-Video game / 1566-Doom / 2352-Doom II: Hell on Earth / \n",
"3-Concert / 5-Dance / 16-Performance art / 68-Lighting / \n",
"323-Toddler / \n",
"616-The Bible / \n",
"31-Disc jockey / \n",
"5-Dance / 49-School / 95-Talent show / \n",
"36-Piano / \n",
"22-Nature / 86-Plant / 3177-Tulip / \n",
"1-Vehicle / 4-Car / 114-Engine / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"9-Music video / 96-Soldier / 1882-Warrior / \n",
"6-Animation / 4221-Saiyuki / \n",
"5-Dance / 16-Performance art / \n",
"5-Dance / \n",
"995-Attack on Titan / \n",
"7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / 150-Violin / 639-Fiddle / \n",
"1-Vehicle / 8-Football / 350-Circus / 788-Juggling / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 127-Xbox 360 / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / 394-Map / 704-Call of Duty: World at War / \n",
"49-School / 410-Teacher / 845-Classroom / \n",
"125-Television / \n",
"0-Games / 6-Animation / 1647-Kidō Senshi Gundam: Senjō no Kizuna / 3289-MS-06 Zaku II / \n",
"20-Trailer / \n",
"10-Animal / 20-Trailer / 48-Pet / 71-Dog / 298-Television advertisement / 1723-Chihuahua / \n",
"3-Concert / 68-Lighting / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / 858-Carl Johnson / \n",
"0-Games / 54-Highlight film / 60-Basketball / 81-Athlete / 168-Basketball moves / \n",
"83-Skateboarding / 108-Skateboard / \n",
"8-Football / \n",
"28-Fashion / 45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 235-Lipstick / 291-Rouge / \n",
"61-Art / 97-Drawing / 479-Graffiti / \n",
"2-Video game / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 422-Call of Duty: Ghosts / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 213-Forest / 380-Assassin's Creed / 1241-Assassin's Creed IV: Black Flag / \n",
"59-Winter sport / 177-Skiing / 355-Comedian / \n",
"0-Games / 6-Animation / 241-Fighting game / 486-M.U.G.E.N / 4231-Slayers / \n",
"1-Vehicle / 4-Car / 84-Snow / 154-Wheel / 158-Tire / \n",
"886-James Bond / \n",
"45-Cosmetics / 56-Hair / \n",
"615-Poker / \n",
"1162-Dietary supplement / \n",
"473-Ford Mustang / \n",
"6-Animation / 15-Cartoon / 97-Drawing / 183-Manga / 1556-Dōjinshi / \n",
"0-Games / 2-Video game / 2265-Subway Surfers / \n",
"12-Food / \n",
"25-Toy / \n",
"5-Dance / 16-Performance art / 98-Festival / 306-High school / \n",
"0-Games / 19-PC game / 271-Tank / 409-World of Tanks / \n",
"10-Animal / 80-Horse / 247-Stallion / 438-Mare / 3772-Paso Fino / \n",
"9-Music video / \n",
"3-Concert / 16-Performance art / \n",
"12-Food / 224-Rapid transit / \n",
"12-Food / 26-Cooking / 32-Recipe / 210-Cookware and bakeware / 2563-Gravy / \n",
"25-Toy / 140-LEGO / 884-Lego minifigure / 1882-Warrior / \n",
"75-Wrestling / \n",
"2-Video game / 61-Art / \n",
"199-Wii / 298-Television advertisement / \n",
"39-Drums / \n",
"10-Animal / 91-Fish / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 118-Grand Theft Auto V / 176-PlayStation 4 / 330-Money / 601-Shark / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 651-Stock car racing / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / \n",
"3-Concert / 5-Dance / 16-Performance art / 38-Orchestra / \n",
"444-Family / \n",
"20-Trailer / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 1723-Chihuahua / \n",
"5-Dance / 78-Wedding / 153-Dress / 190-Bride / 421-Gown / \n",
"1-Vehicle / 4-Car / 114-Engine / 536-Glass / \n",
"20-Trailer / 298-Television advertisement / 337-Advertising / 1172-VHS / \n",
"0-Games / 2-Video game / 366-Dragon / \n",
"6-Animation / 15-Cartoon / 851-Digimon / \n",
"0-Games / 35-Minecraft / \n",
"13-Musical ensemble / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"125-Television / 298-Television advertisement / 337-Advertising / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 105-Motorcycling / 927-Supermoto / 2133-Road racing / 3553-Track racing / \n",
"10-Animal / 540-Insect / 775-Bee / 1110-Cell / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 179-Kitchen / 215-Eating / 262-Ice / 295-Chocolate / 437-Cream / 684-Ice cream / 770-Banana / 2699-Chocolate chip / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / \n",
"117-Boxing / \n",
"0-Games / 2-Video game / 328-Simulation video game / 4187-MechWarrior Online / \n",
"45-Cosmetics / 148-Painting / 303-Paint / 385-Face / 855-Hulk / \n",
"1-Vehicle / 86-Plant / 124-Tractor / 156-Agriculture / 174-Farm / 459-Plough / 741-Mower / 984-Silage / 1037-Grass / \n",
"14-Guitar / 63-Acoustic guitar / 121-Photography / 722-Waterfall / \n",
"93-Comedy / \n",
"0-Games / 6-Animation / 1338-Devil / 1527-Pretty Cure / \n",
"860-Christian Church / \n",
"0-Games / 2-Video game / 35-Minecraft / 55-Video game console / 111-PlayStation 3 / 127-Xbox 360 / 181-Xbox / 314-Xbox One / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 17-Racing / 34-Action-adventure game / 118-Grand Theft Auto V / 297-Grand Theft Auto IV / 787-Monster truck / 1077-Lightning McQueen / \n",
"774-Galaxy / \n",
"356-Star / \n",
"0-Games / 2-Video game / 116-Pokémon / \n",
"1-Vehicle / 27-Motorcycle / \n",
"12-Food / 282-Skin / 385-Face / 641-Mask / \n",
"0-Games / 2-Video game / \n",
"2808-Printed circuit board / \n",
"20-Trailer / \n",
"1-Vehicle / 4-Car / \n",
"150-Violin / \n",
"0-Games / 51-Strategy video game / 373-Clash of Clans / \n",
"13-Musical ensemble / \n",
"416-Apartment / \n",
"5-Dance / 98-Festival / \n",
"10-Animal / 48-Pet / 71-Dog / \n",
"1-Vehicle / 1699-Cart / 3573-Golf cart / \n",
"0-Games / 2-Video game / 6-Animation / 3426-Inazuma Eleven GO / \n",
"3-Concert / 5-Dance / 31-Disc jockey / 68-Lighting / 133-Nightclub / \n",
"93-Comedy / \n",
"7-Musician / 30-Drummer / 39-Drums / 44-Drums / 296-Saxophone / \n",
"6-Animation / 15-Cartoon / 755-Hetalia: Axis Powers / 4671-Little Busters! / \n",
"6-Animation / 15-Cartoon / \n",
"10-Animal / 80-Horse / 290-Pony / \n",
"10-Animal / 80-Horse / \n",
"2009-Electromagnetic coil / 3806-Tesla coil / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 71-Dog / 217-Hunting / 280-Wildlife / 2673-Moose / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 1040-Street racing / \n",
"6-Animation / 15-Cartoon / 126-The Walt Disney Company / \n",
"0-Games / 159-World of Warcraft / 171-Warcraft / 1213-World of Warcraft: Wrath of the Lich King / \n",
"45-Cosmetics / 195-Mascara / \n",
"47-Personal computer / 131-Computer / 497-Macintosh / 900-MacBook / \n",
"93-Comedy / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / \n",
"122-Weight training / 214-Muscle / \n",
"9-Music video / \n",
"5-Dance / 95-Talent show / \n",
"5-Dance / 16-Performance art / 227-Water / \n",
"1151-Temple / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 6-Animation / 123-Naruto / 244-Goku / 591-Naruto: Ultimate Ninja / \n",
"0-Games / 79-American football / 90-Sports game / 561-Madden NFL / 2916-Madden NFL 12 / \n",
"8-Football / 425-City / \n",
"357-DVD / \n",
"3-Concert / 7-Musician / 9-Music video / 13-Musical ensemble / 46-Choir / \n",
"0-Games / 2-Video game / 2708-Paper Mario / \n",
"10-Animal / \n",
"2-Video game / 31-Disc jockey / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 231-Supercar / 352-Need for Speed / 1770-Need for Speed: Hot Pursuit / 2545-Need for Speed III: Hot Pursuit / \n",
"3-Concert / \n",
"0-Games / 9-Music video / 35-Minecraft / 321-Roller coaster / 334-Amusement ride / \n",
"1-Vehicle / 22-Nature / 87-Boat / 164-Beach / 233-Ocean / 423-Coast / \n",
"209-University / \n",
"293-Slide show / \n",
"25-Toy / 126-The Walt Disney Company / 200-Doll / 417-Play-Doh / 996-Princess / 1219-Ariel / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 167-Four-wheel drive / \n",
"1-Vehicle / 124-Tractor / 156-Agriculture / 174-Farm / 660-Rural area / \n",
"12-Food / 26-Cooking / 32-Recipe / 274-Meat / 589-Grilling / 649-Oven / 688-Beef / 1256-Smoking / \n",
"1-Vehicle / 4-Car / 70-Driving / 74-Truck / \n",
"6-Animation / 15-Cartoon / 4344-Gakuen Alice / \n",
"28-Fashion / 45-Cosmetics / \n",
"75-Wrestling / 1080-WWE 2K15 / 1582-WWE 2K14 / \n",
"9-Music video / \n",
"6-Animation / 15-Cartoon / 183-Manga / \n",
"10-Animal / 130-Gymnastics / 132-Bird / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / \n",
"5-Dance / 44-Drums / 1579-Circle / \n",
"0-Games / \n",
"38-Orchestra / 3263-Shadow of the Colossus / \n",
"5-Dance / 49-School / 306-High school / \n",
"1-Vehicle / 4-Car / \n",
"1-Vehicle / 4-Car / 395-Camping / 519-Recreational vehicle / \n",
"12-Food / 26-Cooking / 2164-Silk / 2552-Dyeing / \n",
"1-Vehicle / 18-Outdoor recreation / 22-Nature / 67-Cycling / 69-Bicycle / 197-Trail / 220-Mountain bike / 284-Mountain biking / \n",
"170-Home improvement / 353-Wall / 879-Ceiling / 1350-Plaster / 1671-Drywall / \n",
"3-Concert / 16-Performance art / \n",
"6-Animation / 15-Cartoon / \n",
"809-Logo / \n",
"18-Outdoor recreation / 67-Cycling / 69-Bicycle / 83-Skateboarding / 108-Skateboard / 221-Skatepark / 377-BMX bike / \n",
"263-Light / \n",
"1-Vehicle / 50-Aircraft / 151-Landing / 254-Runway / 262-Ice / \n",
"1-Vehicle / 4-Car / 94-Dashcam / 683-Renault / \n",
"0-Games / 2-Video game / 6-Animation / \n",
"0-Games / 96-Soldier / 264-Star Wars / 714-Jedi / 1050-Star Wars: The Old Republic / \n",
"117-Boxing / \n",
"3-Concert / 7-Musician / 14-Guitar / 24-String instrument / \n",
"28-Fashion / \n",
"75-Wrestling / \n",
"0-Games / 2-Video game / 8-Football / 90-Sports game / 180-FIFA 15 / \n",
"9-Music video / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 3060-Chameleon / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 76-Ball / 81-Athlete / 1037-Grass / \n",
"25-Toy / 200-Doll / 4237-Ball-jointed doll / \n",
"9-Music video / 1396-Lara Croft / \n",
"2-Video game / 34-Action-adventure game / 380-Assassin's Creed / 1241-Assassin's Creed IV: Black Flag / \n",
"5-Dance / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 210-Cookware and bakeware / 239-Roasting / 274-Meat / 454-Sauce / 589-Grilling / 688-Beef / 720-Wine / 862-Pork / 1115-Steak / 2125-Fillet / \n",
"3-Concert / 7-Musician / 46-Choir / \n",
"12-Food / 120-Winter / 137-Tree / 348-Fruit / \n",
"0-Games / 116-Pokémon / 186-Pokémon / 690-Pokémon X and Y / \n",
"8-Football / \n",
"1-Vehicle / 27-Motorcycle / 278-GoPro / \n",
"121-Photography / 633-Photograph / \n",
"0-Games / 2-Video game / 686-Diablo III / 872-Diablo / \n",
"0-Games / 2-Video game / 141-Microsoft Windows / \n",
"14-Guitar / 63-Acoustic guitar / 361-Flamenco / \n",
"20-Trailer / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 38-Orchestra / \n",
"12-Food / 52-Dish / 58-Cuisine / 1539-Ramen / 2230-Buffet / \n",
"0-Games / 2-Video game / 257-Counter-Strike / 425-City / 702-Counter-Strike: Source / \n",
"170-Home improvement / 276-Room / 308-Ship / 605-Cruise ship / 652-Bathroom / 1402-Suite / 2729-Balcony / \n",
"9-Music video / \n",
"3-Concert / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 179-Kitchen / 1108-Mushroom / 1230-Italian food / \n",
"10-Animal / 22-Nature / 280-Wildlife / 537-Zoo / \n",
"0-Games / 319-Robot / 1885-Robotics / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 557-Salad / 1748-Carrot / 2588-Mayonnaise / 3205-Beetroot / \n",
"122-Weight training / 214-Muscle / 1162-Dietary supplement / \n",
"60-Basketball / \n",
"242-Paper / 370-Origami / \n",
"615-Poker / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / \n",
"10-Animal / 91-Fish / 258-Aquarium / 2090-Siamese fighting fish / 3018-Betta / \n",
"0-Games / 25-Toy / 489-Board game / 922-Tabletop game / \n",
"1-Vehicle / 4-Car / 11-Motorsport / \n",
"259-Fire / 616-The Bible / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 38-Orchestra / \n",
"457-Knitting / 696-T-shirt / 907-Necklace / 1096-Scarf / \n",
"9-Music video / \n",
"5-Dance / \n",
"9-Music video / 20-Trailer / 925-Brain / \n",
"0-Games / 2-Video game / 159-World of Warcraft / 171-Warcraft / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 790-Nissan GT-R / 952-Chevrolet Corvette / \n",
"313-Book / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 2244-Autocross / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 352-Need for Speed / 621-Asus / 875-Need for Speed: Most Wanted / 1577-Need for Speed: Most Wanted / \n",
"3-Concert / \n",
"56-Hair / 106-Hairstyle / 340-Afro-textured hair / 405-Braid / 798-Shampoo / 1812-Hair twists / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 1013-Walking / 2019-Dental braces / \n",
"36-Piano / 53-Keyboard / 104-Pianist / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 542-Potato / 2955-Waffle / 4415-Mashed potato / \n",
"8-Football / 21-Stadium / 60-Basketball / \n",
"28-Fashion / \n",
"8-Football / \n",
"20-Trailer / \n",
"68-Lighting / 92-Radio-controlled model / 202-Radio-controlled car / 263-Light / \n",
"1-Vehicle / 20-Trailer / 74-Truck / 412-Monster / 787-Monster truck / 1014-Movieclips / \n",
"3-Concert / \n",
"6-Animation / 15-Cartoon / 126-The Walt Disney Company / 1172-VHS / \n",
"12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / 239-Roasting / 1295-Stir frying / \n",
"18-Outdoor recreation / 42-Fishing / 478-Fishing rod / 510-Fishing lure / 844-Bass / 3204-Cod / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 517-Audi / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 76-Ball / \n",
"1642-Cigarette / \n",
"489-Board game / 799-Chess / \n",
"45-Cosmetics / 780-Fat / \n",
"10-Animal / 22-Nature / 48-Pet / 136-Gardening / 588-Reptile / 763-Snake / \n",
"3-Concert / \n",
"10-Animal / 48-Pet / 71-Dog / 2049-Police dog / 2758-Malinois / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 2140-Tony Hawk's / \n",
"358-Rugby football / \n",
"10-Animal / 48-Pet / 166-Cat / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 8-Football / 21-Stadium / 90-Sports game / 97-Drawing / 180-FIFA 15 / 848-FIFA 12 / \n",
"6-Animation / 123-Naruto / 183-Manga / 622-Sakura Haruno / \n",
"6-Animation / 15-Cartoon / 123-Naruto / 285-Sasuke Uchiha / \n",
"0-Games / 8-Football / \n",
"10-Animal / 132-Bird / \n",
"1-Vehicle / 4-Car / 40-Road / 78-Wedding / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 85-Combat / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / 1970-Axe / \n",
"18-Outdoor recreation / 33-Weapon / 42-Fishing / 227-Water / \n",
"1-Vehicle / 4-Car / \n",
"9-Music video / \n",
"79-American football / \n",
"28-Fashion / 768-Diamond / \n",
"10-Animal / 48-Pet / 132-Bird / 598-Parrot / \n",
"463-Cricket / 1190-Batting / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / \n",
"322-Mixtape / \n",
"444-Family / \n",
"56-Hair / 106-Hairstyle / \n",
"9-Music video / 14-Guitar / \n",
"361-Flamenco / \n",
"1-Vehicle / 11-Motorsport / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 472-Boeing 737 / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / \n",
"0-Games / 8-Football / \n",
"125-Television / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 704-Call of Duty: World at War / 1248-Call of Duty: Zombies / \n",
"5-Dance / \n",
"20-Trailer / \n",
"0-Games / 985-Strawberry / 1211-Transformice / \n",
"0-Games / 241-Fighting game / 486-M.U.G.E.N / 1618-Rayman / \n",
"12-Food / 215-Eating / \n",
"5-Dance / 430-Ballroom dance / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 84-Snow / 262-Ice / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"28-Fashion / 757-Boot / \n"
]
}
],
"source": [
"i = 0\n",
"for item in train_labels.iterrows():\n",
" labels = item[1][\"labels\"]\n",
" for l in sorted(map(int, labels.split())):\n",
" print \"%d-%s\" % (l, label[l]), \"/\",\n",
" print \"\"\n",
" i += 1\n",
" if i >= 10000:\n",
" break"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### statistics for label-sample coverage"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"i = 0\n",
"label_sample_set = {}\n",
"for item in train_labels.iterrows():\n",
" labels = item[1][\"labels\"]\n",
" for l in sorted(map(int, labels.split())):\n",
" if l not in label_sample_set:\n",
" label_sample_set[l] = set()\n",
" label_sample_set[l].add(i)\n",
" i += 1"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"x = range(4716)\n",
"y = [0] * 4716\n",
"s = set()\n",
"for l in x:\n",
" s.update(label_sample_set[l])\n",
" y[l] = len(s)"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAhIAAAFkCAYAAAB1rtL+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3XuU3WV97/H3NxcCCTCgkUQuBRREFEQygsR6QaiiVTm1\nttJRtIrVUjmnOh6vWKVFrce6wFOORunxKKTUseg6KrW6okEXHMVAnQHUSvBCMNwSwm1CSAIJ+Z4/\nnj1kz87MZPYvmdm/ybxfa/3Wnv38nt9+nnlmkt9nnt8tMhNJkqQqZnS6A5IkaeoySEiSpMoMEpIk\nqTKDhCRJqswgIUmSKjNISJKkygwSkiSpMoOEJEmqzCAhSZIqM0hIkqTK2g4SEfGiiLgqIu6KiG0R\nceY4tjk1IvojYnNE/Coi/rxadyVJUp1UmZGYB9wEnAfs9EEdEXEE8G3gauAE4B+BL0bEyyq0LUmS\naiR25aFdEbEN+KPMvGqMOp8CXpmZz2kq6wO6MvMPKzcuSZI6bjLOkTgFWN5StgxYPAltS5KkCTRr\nEtpYCKxtKVsL7B8RczLz0dYNIuLJwBnA7cDmCe+hJEl7jr2BI4BlmXn/RDc2GUFiJNF4He24yhnA\nv0xSXyRJ2hO9EfjKRDcyGUFiDbCgpewgYH1mPjbKNrcDXHHFFRx77LET2DU16+3t5TOf+UynuzGt\nOObjkwmPPw5bt5bX5q+by1rXbdu24/pLL+3lLW/5TFvb7Kyd8dYZ6bPH21YnzZgxfJk5EyLG/3rv\nvb0cfPBnmDlz+OeMtc2sWeXroW1mzixlze2P9lmt/R1tXWt5a/929j2PtE3z+9Y6c+fCU54y8T+v\nW265hbPPPhsa+9KJNhlB4ifAK1vKXt4oH81mgGOPPZZFixZNVL/Uoqury/GeZJM55kM74y1b4LHH\nyjL09Xheh5bHHis7ui1b4NFHhy+bNm3/7JGWLVvKtkPbD702f3ZrG0M71d2ni/e9b+QxnzVr15e9\n9hr+fmgHOBHLSJ89cybMnr19XXOd1h3z0OtYZREjDlVbzjyzi6uu8v+WDpiUUwPaDhIRMQ84iu2H\nJ54WEScAD2TmHRHxSeDgzBy6V8QXgP/auHrjS8DpwJ8AXrGhaSFz+0538+bhrw89BNddt2P5RL1u\n27b7vq/Zs2HOnOHL3nuX17322nHZb7+yzezZ23dsQ1/Pnl3qNL82rxupfnPZ0M5zrJ3sUJ23vhX6\n+nZct7t2mtJ0U2VG4nnADynnNyRwUaP8cuAcysmVhw1VzszbI+JVwMXAXwN3Am/LzNYrOaRJtXVr\n+Qt6aHnoobI88khZNmwoS/POeGhp3jkP/SW+aRNs3Lj9dePG7duPdZX17//+yOUzZmzfMTfvpEd6\n7eoae33zzr51h928s29+P7QzH1qay4amd6eiOXPgSU/qdC+kPUfbQSIzr2GMy0Yz862jbNPdblvS\nkG3bhu+gN24sO/uh16Gd/oMPbg8ErcuGDdt39Js2lVmCndlrL9hnn7Iz3mefkXfMc+aUHfnCheUY\n6D77bH/db7/t24+0c//gB+Hznx95/axOnQotSW3wvyo9oaenZ7d8zubNsH49PPwwPPDA9h35xo2j\n/3W/fj3cd1/ZZsOG8jr09SOPlGPm47HffnDAAcOXww+H5zynrJs7d/tOvnmZOxf23x8OPBDmzYN9\n9y2vE70zP/fcHp71rIltQ8Ptrt9zjZ9jvmfbpTtbTpSIWAT09/f3e/JfDWzYUALBXXfB2rXbp/8f\negjWrCmzAA8+CHfcUZaHHhr9syLKX9vNf3nvvXfZaR90UNnZDy377lte580bvtOfN68sQ6Fg6P28\neWXaXZKms4GBAbq7uwG6M3NgottzRmKayyyzAffeC/fcAytXwk03werV8Lvfldf163fcbvbs8tf+\nggXw5CeXrxcvhje9qQSCAw8sQeDAA8vS1VV2+nvtNXWPrUuSdmSQ2INt2QJ3311Cwm23wZ13wrp1\nZXbh7rvLDMNttw0PCjNmwLHHwpFHwkteAr/3e3DooSUsLFwIT31qCQb+5S9JAoPElJVZwsCqVWXm\nYM0auP32EgxWrSqHGDZsGL7NfvuVm6E86UklEJxyCrz+9fD0p5fyhQvhiCPK4QNJksbDIDEFZMJv\nfgPXXAM/+9n25cEHt9eZN6+EgKc9DU4/vXx94IFwyCElIBxyCMyf36nvQJK0pzJI1ERmOUnxnnvK\neQk//nEJD7/5TZlpuO++ctjh6KPLFQjveQ8cfzwcdVS5KmHffTv9HUiSpiODRIesWQPf/jb89Kfw\ny1+WkxzXrdu+/sAD4bjj4IQT4Mwz4bnPhZe+1MAgSaoXg8QEe/RR+PWvob+/HI6480649dbydWYJ\nC8cdB6edBs96Fhx8cDkUcfjhntAoSao/g8RutG1bOdnx+uvL8xN+8hP4z//cfjOlpz+9XAHx/OfD\nO94Br3kNHHbY2J8pSVKdGSR20Y03wmc/C7/6FQwMlLs3AhxzTLmvwlvfCieeWM5lWLiws32VJGl3\nM0hUdO+98MlPlhAxfz6cemo5l+E5z4Hubq+QkCRNDwaJNmzbBldeCZ/7HPzoR+Uqine/G/72b8s9\nGiRJmm4MEuOwdm15QuNll5WbP51yCvzTP8GrX11u7CRJ0nRlkBhFJnz1q/CRj8Bvf1uuoHjVq+BL\nXypXWEiSJJjR6Q7UUSZ84APwhjeUGYcrryw3hfrGNwwRkiQ1c0aixU03lbtG/vCH8NGPwt/9Xad7\nJElSfTkj0ZAJ55xTLtVcvRq+8x1DhCRJO2OQoFyNcc458OUvw4c/XO46+cpXdrpXkiTV37Q/tLFi\nRbmE8/rryz0hzjuv0z2SJGnqmLYzEvfcU86FWLwY7r8f+voMEZIktWtazkgsXw5vfCM88AC87W1w\n0UXQ1dXpXkmSNPVMuyBx//3wp38KRxwB3/teeUy3JEmqptKhjYg4LyJWRcSmiFgRESeNUXdWRHw0\nIn7TqH9jRJxRvcvVrV1bnrw5a1a5N4QhQpKkXdN2kIiIs4CLgAuAE4GbgWURMdpjqj4BvB04DzgW\nuBT4RkRM6m589Wo4/ni44w748Y/h6KMns3VJkvZMVWYkeoFLM3NpZq4EzgU2AueMUv9s4BOZuSwz\nb8/MLwDfAf57pR5XsHx5CQ5bt5YbTT3jGZPVsiRJe7a2zpGIiNlAN/D3Q2WZmRGxHFg8ymZzgEdb\nyjYBL2yn7aquvx7e9CaYOxd+8xt40pMmo1VJkqaHdmck5gMzgbUt5WuBhaNsswx4T0QcFcXLgD8G\nJvy5mevWlWdj7LcfLFtmiJAkaXfbXfeRCCBHWfcu4NfASsrMxCXAl4DHd1Pbw9xxB/zlX8ILXwgL\nFsDGjbBkCZx88kS0JknS9Nbu5Z/3UQLAgpbyg9hxlgKAzLwP+OOI2At4cmbeExH/A1i1s8Z6e3vp\narnBQ09PDz09PaNu8/KXw8qV8Cd/ApdeWh79ffDBO2tJkqSpp6+vj76+vmFlg4ODk9qHyBxtImGU\nDSJWANdn5rsa7wNYDVySmZ8ex/azgV8CX83Mj4xSZxHQ39/fz6JFi8bdt8FBOOAA+PSn4b3vHfdm\nkiTtMQYGBuju7gbozsyBiW6vyqGNi4F3RMSbI+KZwBeAucBlABGxNCKeOBkzIk6OiNdGxJER8SLg\nu5RDITsNHe249VY47rjy9RkduUuFJEnTT9t3tszMKxv3jLiQcojjJuCMzFzXqHIosLVpk72BjwNH\nAhuAfwfOzsz1u9LxZvffD897Xrky44c/LPeLkCRJE6/SLbIzcwmwZJR1p7W8vxZ4dpV2xmPbtjIT\nsWEDfO1rcOqpE9WSJElqNeWf/rl0KaxZU15f8YpO90aSpOllygeJf/3Xcp+IN7yh0z2RJGn6mfJB\nYu3aMhMxc2aneyJJ0vQzpYPEo4/CjTf6FE9JkjplSgeJr32tvL74xZ3thyRJ09WUDhKf+hQ8//nw\nohd1uieSJE1PUzZI/OpX8ItflCAhSZI6Y8oGiU98oryefXZn+yFJ0nQ2ZYPENdfA298OJ53U6Z5I\nkjR9Tckgcddd8Lvfwcte1umeSJI0vU3JIPHtb8OMGfDSl3a6J5IkTW9TMkjceiscfDDMn9/pnkiS\nNL1NySDxrW/BM5/Z6V5IkqQpFyTWrIHbboPXv77TPZEkSVMuSPz85+V18eLO9kOSJE3BIHHzzTB7\ntoc2JEmqgykXJL76VXj2s2HWrE73RJIkTakgsXkz9PfDG97Q6Z5IkiSYYkFi9eryeswxne2HJEkq\nplSQ+PCHy+uxx3a2H5IkqZhSQeL734c3vxmOOqrTPZEkSTCFgsS2bbB+PbzwhRDR6d5IkiSYQkHi\nP/4DMuGAAzrdE0mSNGTKBInvfre8Pu95ne2HJEnarlKQiIjzImJVRGyKiBURcdJO6r87IlZGxMaI\nWB0RF0fEnHbavP9+OO44OPLIKj2WJEkToe0gERFnARcBFwAnAjcDyyJixGdxRsQbgE826j8TOAc4\nC/hEO+0+8AA8+cnt9laSJE2kKjMSvcClmbk0M1cC5wIbKQFhJIuBH2Xmv2bm6sxcDvQBJ7fT6I9/\nbJCQJKlu2goSETEb6AauHirLzASWUwLDSK4DuocOf0TE04A/BP69nbYffhgOPLCdLSRJ0kRr94kV\n84GZwNqW8rXAiPebzMy+xmGPH0VENLb/QmZ+aryN/uhH5dDGH/xBm72VJEkTanc9+iqAHHFFxKnA\n+ZRDIDcARwGXRMQ9mfnxsT60t7eXrq4u+vth773hiisgs4eenp7d1G1Jkqauvr4++vr6hpUNDg5O\nah+iHJkYZ+VyaGMj8LrMvKqp/DKgKzNfO8I21wI/ycwPNJW9kXKexb6jtLMI6O/v72fRokU8//kl\nSFxzzbi7KknStDQwMEB3dzdAd2YOTHR7bZ0jkZlbgH7g9KGyxuGK0ynnQoxkLrCtpWxbY9Nx3aNy\n/XrvHyFJUh1VObRxMXB5RPRTDlX0UsLCZQARsRS4MzPPb9T/N6A3Im4CrgeOBi4EvpXjnA5Zvx72\n379CTyVJ0oRqO0hk5pWNkycvBBYANwFnZOa6RpVDga1Nm3yMMgPxMeAQYB1wFfA3423TICFJUj1V\nOtkyM5cAS0ZZd1rL+6EQ8bEqbT3+OGzYYJCQJKmOav+sjdtvL68LFnS0G5IkaQS1DxJf/GJ5Pbmt\n+2BKkqTJUPsgceON8KIXwUEHdbonkiSpVe2DxPr1cNRRne6FJEkayZQIEp5oKUlSPdU6SGTCunXQ\n1dXpnkiSpJHUOkjcfTfcey+cdFKneyJJkkZS6yDx0EPl9bDDOtsPSZI0sloHiY0by+u+Iz7aS5Ik\ndZpBQpIkVVbrIPHII+V1v/062w9JkjSyWgeJhx+GvfaCffbpdE8kSdJIah0kNm4shzUiOt0TSZI0\nkloHiUcfdTZCkqQ6M0hIkqTKah0kNm82SEiSVGe1DhK//a1BQpKkOqt1kIiATZs63QtJkjSaWgeJ\nzZvh5JM73QtJkjSaWgeJTZtg3rxO90KSJI2m9kFi7txO90KSJI2m9kHCGQlJkurLICFJkiqrFCQi\n4ryIWBURmyJiRUScNEbdH0bEthGWf9tZO5s3GyQkSaqztoNERJwFXARcAJwI3Awsi4j5o2zyWmBh\n03Ic8Dhw5XjaM0hIklRfVWYkeoFLM3NpZq4EzgU2AueMVDkzH8rMe4cW4OXAI8DXx9OYQUKSpPpq\nK0hExGygG7h6qCwzE1gOLB7nx5wD9GXmuG41ZZCQJKm+2p2RmA/MBNa2lK+lHLYYU0ScDDwb+OJ4\nGzRISJJUX7vrqo0Achz13gb8IjP7x/vBT31q5T5JkqQJNqvN+vdRTpRc0FJ+EDvOUgwTEfsAZwF/\nM/7mennnO7uYPXt7SU9PDz09PeP/CEmS9lB9fX309fUNKxscHJzUPkQ5xaGNDSJWANdn5rsa7wNY\nDVySmZ8eY7u3AEuAQzLzwZ20sQjoh342b17EnDltdVGSpGlrYGCA7u5ugO7MHJjo9tqdkQC4GLg8\nIvqBGyhXccwFLgOIiKXAnZl5fst2bwO+ubMQ0WqvvSr0UJIkTYq2g0RmXtm4Z8SFlEMcNwFnZOa6\nRpVDga3N20TE0cALgJe109acOeVR4pIkqZ6qzEiQmUsohylGWnfaCGW/plzt0RYPaUiSVG+1ftaG\nQUKSpHozSEiSpMpqHSS8h4QkSfVW6yBx8MGd7oEkSRpLrYOEhzYkSaq3WgcJ7yEhSVK9GSQkSVJl\ntQ4SHtqQJKneah0k9t670z2QJEljqXWQ2H//TvdAkiSNpdZBYlalG3hLkqTJUusgMbPtp3NIkqTJ\nVOsgMaPWvZMkSbXeVTsjIUlSvdU6SDgjIUlSvdV6V22QkCSp3mq9q/bQhiRJ9VbrIBHR6R5IkqSx\n1DpIOCMhSVK91TpIeI6EJEn1VutdtTMSkiTVW62DhDMSkiTVW6131c5ISJJUb7UOEs5ISJJUb5V2\n1RFxXkSsiohNEbEiIk7aSf2uiPhcRNzd2GZlRLxip50zSEiSVGttP6g7Is4CLgLeAdwA9ALLIuIZ\nmXnfCPVnA8uBNcAfA3cDhwMP7awtD21IklRvbQcJSnC4NDOXAkTEucCrgHOAfxih/tuAA4BTMvPx\nRtnq8TTkjIQkSfXW1q66MbvQDVw9VJaZSZlxWDzKZq8BfgIsiYg1EfHziPhQROy0bWckJEmqt3Zn\nJOYDM4G1LeVrgWNG2eZpwGnAFcArgaOBJY3P+fhYjTkjIUlSvVU5tDGSAHKUdTMoQeMdjdmLGyPi\nEOC97CRIfPKTvXz5y13Dynp6eujp6dn1HkuSNMX19fXR19c3rGxwcHBS+9BukLgPeBxY0FJ+EDvO\nUgy5B3isESKG3AIsjIhZmbl1tMY++tHP8OpXL2qzi5IkTQ8j/XE9MDBAd3f3pPWhrYMHmbkF6AdO\nHyqLiGi8v26UzX4MHNVSdgxwz1ghAjy0IUlS3VXZVV8MvCMi3hwRzwS+AMwFLgOIiKUR8fdN9T8P\nPDki/jEijo6IVwEfAj67s4Y82VKSpHpr+xyJzLwyIuYDF1IOcdwEnJGZ6xpVDgW2NtW/MyJeDnwG\nuBm4q/H1SJeKDuOMhCRJ9VbpZMvMXEK58mKkdaeNUHY98IJ22zFISJJUb7XeVRskJEmqt1rvqg0S\nkiTVW6131QYJSZLqrda7aoOEJEn1VutdtUFCkqR6q/Wu2iAhSVK91XpXbZCQJKnear2rNkhIklRv\ntd5VR3S6B5IkaSy1DhKSJKneDBKSJKkyg4QkSarMICFJkiozSEiSpMoMEpIkqTKDhCRJqswgIUmS\nKjNISJKkygwSkiSpMoOEJEmqzCAhSZIqM0hIkqTKDBKSJKmySkEiIs6LiFURsSkiVkTESWPU/fOI\n2BYRjzdet0XExupdliRJddF2kIiIs4CLgAuAE4GbgWURMX+MzQaBhU3L4e13VZIk1U2VGYle4NLM\nXJqZK4FzgY3AOWNsk5m5LjPvbSzrqnRWkiTVS1tBIiJmA93A1UNlmZnAcmDxGJvuGxG3R8TqiPhm\nRDyrUm8lSVKttDsjMR+YCaxtKV9LOWQxklspsxVnAm9stHldRBzSZtuSJKlmZu2mzwkgR1qRmSuA\nFU9UjPgJcAvwDsp5FpIkaYpqN0jcBzwOLGgpP4gdZylGlJlbI+JG4Kid1e3t7aWrq2tYWU9PDz09\nPePrrSRJe7C+vj76+vqGlQ0ODk5qH6Kc4tDGBhErgOsz812N9wGsBi7JzE+PY/sZwC+A72Tme0ep\nswjo7+/vZ9GiRW31T5Kk6WxgYIDu7m6A7swcmOj2qhzauBi4PCL6gRsoV3HMBS4DiIilwJ2ZeX7j\n/UcohzZ+AxwAvJ9y+ecXd7XzkiSps9oOEpl5ZeOeERdSDnHcBJzRdEnnocDWpk0OBP6JcjLmg0A/\nsLhx6agkSZrCKp1smZlLgCWjrDut5f17gPdUaUeSJNWbz9qQJEmVGSQkSVJlBglJklSZQUKSJFVm\nkJAkSZUZJCRJUmUGCUmSVJlBQpIkVWaQkCRJlRkkJElSZQYJSZJUmUFCkiRVZpCQJEmVGSQkSVJl\nBglJklSZQUKSJFVmkJAkSZUZJCRJUmUGCUmSVJlBQpIkVWaQkCRJlRkkJElSZQYJSZJUmUFCkiRV\nVilIRMR5EbEqIjZFxIqIOGmc2/1ZRGyLiP9bpV1JklQvbQeJiDgLuAi4ADgRuBlYFhHzd7Ld4cCn\ngWsr9FOSJNVQlRmJXuDSzFyamSuBc4GNwDmjbRARM4ArgI8Cq6p0VJIk1U9bQSIiZgPdwNVDZZmZ\nwHJg8RibXgDcm5lfrtJJSZJUT7ParD8fmAmsbSlfCxwz0gYR8fvAW4ET2u6dJEmqtXaDxGgCyB0K\nI/YF/hl4e2Y+2O6H9vb20tXVNaysp6eHnp6eqv2UJGmP0dfXR19f37CywcHBSe1DlCMT46xcDm1s\nBF6XmVc1lV8GdGXma1vqnwAMAI9TwgZsP5zyOHBMZu5wzkRELAL6+/v7WbRo0fi/G0mSprmBgQG6\nu7sBujNzYKLba+sciczcAvQDpw+VRUQ03l83wia3AMcDz6Uc2jgBuAr4QePrOyr1WpIk1UKVQxsX\nA5dHRD9wA+UqjrnAZQARsRS4MzPPz8zHgF82bxwRD1HO0bxlVzouSZI6r+0gkZlXNu4ZcSGwALgJ\nOCMz1zWqHAps3X1dlCRJdVXpZMvMXAIsGWXdaTvZ9q1V2pQkSfXjszYkSVJlBglJklSZQUKSJFVm\nkJAkSZUZJCRJUmUGCUmSVJlBQpIkVWaQkCRJlRkkJElSZQYJSZJUmUFCkiRVZpCQJEmVGSQkSVJl\nBglJklSZQUKSJFVmkJAkSZUZJCRJUmUGCUmSVJlBQpIkVWaQkCRJlRkkJElSZQYJSZJUmUFCkiRV\nVilIRMR5EbEqIjZFxIqIOGmMuq+NiP+IiAcjYkNE3BgRZ1fvsiRJqou2g0REnAVcBFwAnAjcDCyL\niPmjbHI/8HHgFOB44MvAlyPiZZV6LEmSaqPKjEQvcGlmLs3MlcC5wEbgnJEqZ+a1mfmtzLw1M1dl\n5iXAz4AXVu61JEmqhbaCRETMBrqBq4fKMjOB5cDicX7G6cAzgGvaaVuSJNXPrDbrzwdmAmtbytcC\nx4y2UUTsD9wFzAG2Au/MzB+02bYkSaqZdoPEaALIMdY/DJwA7AucDnwmIm7LzGt3U/uSJKkD2g0S\n9wGPAwtayg9ix1mKJzQOf9zWePuziHgW8CFgzCDR29tLV1fXsLKenh56enra7LYkSXuevr4++vr6\nhpUNDg5Oah+i7OPb2CBiBXB9Zr6r8T6A1cAlmfnpcX7G/wGOzMzTRlm/COjv7+9n0aJFbfVPkqTp\nbGBggO7uboDuzByY6PaqHNq4GLg8IvqBGyhXccwFLgOIiKXAnZl5fuP9B4GfAr+lnCPxKuBsytUe\nkiRpCms7SGTmlY17RlxIOcRxE3BGZq5rVDmUckLlkHnA5xrlm4CVwBsz8+u70nFJktR5lU62zMwl\nwJJR1p3W8v4jwEeqtCNJkurNZ21IkqTKDBKSJKkyg4QkSarMICFJkiozSEiSpMoMEpIkqTKDhCRJ\nqswgIUmSKjNISJKkygwSkiSpMoOEJEmqzCAhSZIqM0hIkqTKDBKSJKkyg4QkSarMICFJkiozSEiS\npMoMEpIkqTKDhCRJqswgIUmSKjNISJKkygwSkiSpMoOEJEmqzCAhSZIqqxQkIuK8iFgVEZsiYkVE\nnDRG3b+IiGsj4oHG8v2x6kuSpKmj7SAREWcBFwEXACcCNwPLImL+KJu8BPgKcCpwCnAH8L2IeGqV\nDkuSpPqoMiPRC1yamUszcyVwLrAROGekypn5psz8Qmb+LDN/BfxFo93Tq3ZakiTVQ1tBIiJmA93A\n1UNlmZnAcmDxOD9mHjAbeKCdtiVJUv20OyMxH5gJrG0pXwssHOdnfAq4ixI+JEnSFDZrN31OALnT\nShEfBF4PvCQzH9tZ/d7eXrq6uoaV9fT00NPTU7WfkiTtMfr6+ujr6xtWNjg4OKl9iHJkYpyVy6GN\njcDrMvOqpvLLgK7MfO0Y274XOB84PTNv3Ek7i4D+/v5+Fi1aNO7+SZI03Q0MDNDd3Q3QnZkDE91e\nW4c2MnML0E/TiZIREY331422XUS8D/gwcMbOQoQkSZo6qhzauBi4PCL6gRsoV3HMBS4DiIilwJ2Z\neX7j/fuBC4EeYHVELGh8zobMfGTXui9Jkjqp7SCRmVc27hlxIbAAuIky07CuUeVQYGvTJn9FuUrj\n6y0f9XeNz5AkSVNUpZMtM3MJsGSUdae1vD+yShuSJKn+fNaGJEmqzCAhSZIqM0hIkqTKDBKSJKky\ng4QkSarMICFJkiozSEiSpMoMEpIkqTKDhCRJqswgIUmSKjNISJKkygwSkiSpMoOEJEmqzCAhSZIq\nM0hIkqTKDBKSJKkyg4QkSarMICFJkiozSEiSpMoMEpIkqTKDhCRJqswgIUmSKjNI6Al9fX2d7sK0\n45hPPsd88jnme7ZKQSIizouIVRGxKSJWRMRJY9R9VkR8vVF/W0T8dfXuaiL5j33yOeaTzzGffI75\nnq3tIBERZwEXARcAJwI3A8siYv4om8wFfgt8ALinYj8lSVINVZmR6AUuzcylmbkSOBfYCJwzUuXM\n/GlmfiAzrwQeq95VSZJUN20FiYiYDXQDVw+VZWYCy4HFu7drkiSp7ma1WX8+MBNY21K+Fjhmt/So\n2Bvglltu2Y0fqZ0ZHBxkYGCg092YVhzzyeeYTz7HfHI17Tv3noz22g0Sowkgd9NnARwBcPbZZ+/G\nj9R4dHd3d7oL045jPvkc88nnmHfEEcB1E91Iu0HiPuBxYEFL+UHsOEuxK5YBbwRuBzbvxs+VJGlP\ntzclRCybjMbaChKZuSUi+oHTgasAIiIa7y/ZXZ3KzPuBr+yuz5MkaZqZ8JmIIVUObVwMXN4IFDdQ\nruKYC1wGEBFLgTsz8/zG+9nAsyiHP/YCDomIE4ANmfnbXf4OJElSx0S56KLNjSLeCbyfcojjJuC/\nZeZPG+sueThKAAAGa0lEQVR+ANyemec03h8OrGLHcyiuyczTdqHvkiSpwyoFCUmSJPBZG5IkaRcY\nJCRJUmW1CxLtPBBMw0XEiyLiqoi4q/GAtDNHqHNhRNwdERsj4vsRcVTL+gMj4l8iYjAiHoyIL0bE\nvJY6z4mIaxs/o99FxPsm+nuro4j4UETcEBHrI2JtRHwjIp7RUmdORHwuIu6LiIcbD7A7qKXOYRHx\n7xHxSESsiYh/iIgZLXVOjYj+iNgcEb+KiD+fjO+xbiLi3Ii4ufH7ORgR10XEK5rWO94TrPF7vy0i\nLm4qc9x3o4i4oDHGzcsvm9bXa7wzszYLcBblvhFvBp4JXAo8AMzvdN+mwgK8ArgQ+CPK/T7ObFn/\ngcZ4vgY4Dvgm5YFqezXV+S4wADwPeAHwK+CKpvX7UR6+djlwLPB64BHgLzr9/XdgvL8DvKkxDscD\n36bc+2Sfpjqfb5S9hPKQu+uA/9e0fgbwc8r13scDZwD3Ah9vqnMEsAH4B8odZM8DtgAv6/QYdGDM\nX9X4PT+qsXwceBQ41vGelPE/CbgNuBG4uKnccd+943wB8DPgKZT7NB0EPKmu493xAWsZvBXAPza9\nD+BO4P2d7ttUW4Bt7Bgk7gZ6m97vD2wCXt94f2xjuxOb6pwBbAUWNt7/FeXGZLOa6nwS+GWnv+dO\nL5RbyG8DXtg0vo8Cr22qc0yjzsmN969s/OOd31TnL4EHh8YY+BTws5a2+oDvdPp7rsMC3A+81fGe\n8HHeF7gVOA34IY0g4bhPyFhfAAyMsq52412bQxs+EGxiRcSRwEKGj+964Hq2j+8pwIOZeWPTpssp\nl+4+v6nOtZm5tanOMuCYiOiaoO5PFQdQxuqBxvtuyr1amsf8VmA1w8f855l5X9PnLAO6gGc31Vne\n0tYypvm/i4iYERF/RrmPzU9wvCfa54B/y8wftJQ/D8d9IhzdOEz924i4IiIOa5TX7ve8NkGCsR8I\ntnDyu7PHWUjZyY01vgsp019PyMzHKTvG5jojfQZM459TRATwP4EfZebQscyFwGONwNasdcx3Np6j\n1dk/Iubsat+nmog4LiIepvxVtoTyl9lKHO8J0whszwU+NMLqBTjuu9sK4C2UGeFzgSOBaxvnq9Xu\n93x3PbRrIu3uB4JpuPGM787qRON1Ov+cllDu4PrCcdQd7++0Yz6ylcAJlBmg1wFLI+LFY9R3vHdB\nRBxKCckvy8wt7WyK415JZjY/I+MXEXED8DvKOWmjPX+qY+NdpxmJyXog2HS1hvJLMtb4rmm8f0JE\nzAQObKwbqjPSZ8A0/TlFxGeBPwROzcy7m1atAfaKiP1bNmkd89bxXNC0brQ6BwHrM/OxXen7VJSZ\nWzPztswcyMwPAzcD78LxnijdlJP++iNiS0RsoZzk966IeIwytnMc94mTmYOUE9+Pooa/57UJEo2k\nO/RAMGDYA8Em7eEje6rMXEX5xWke3/0p5z4Mje9PgAMi4sSmTU+nBJAbmuq8uBEwhrwcuLXxyz6t\nNELEfwFempmrW1b3U05UbR7zZwC/x/AxPz4i5jdt93JgELilqc7pDPfyRrnK/2NzcLwnynLKmf/P\npcwEnQD8FLii6estOO4TJiL2BZ5OOWG+fr/nnT47teWM0ddTriJovvzzfuApne7bVFiAeZR/2M+l\nnMH77sb7wxrr398Yz9dQ/mP4JvBrhl/++R3KfwwnAb9POUv7n5vW70/5Zb6cMpV/FuUSord1+vvv\nwHgvoZwF/SJKsh9a9m6pswo4lfKX3Y/Z8TKtmymX3T6Hckx0LfCxpjpHNMb4U5Szs98JPAb8QafH\noANj/gnK4aPDKZcwf5Lyn+ppjvek/hyeuGrDcZ+Q8f008OLG7/kLgO83xuvJdRzvjg/YCAP4Tsr1\nsZsoyeh5ne7TVFko043bKIeImpcvNdX5W0oQ2Eg5Q/eols84gPKXxiBlJ/m/gbktdY4Hrml8xmrg\nvZ3+3js03iON9ePAm5vqzAH+F+XQ3cPA14CDWj7nMMo9KDY0/rF/Cpgxws+2v/Hv4tfAmzr9/Xdo\nzL9IuY/BJsoM2/dohAjHe1J/Dj9geJBw3Hfv+PZRbn2wqfF/7FeAI+s63j60S5IkVVabcyQkSdLU\nY5CQJEmVGSQkSVJlBglJklSZQUKSJFVmkJAkSZUZJCRJUmUGCUmSVJlBQpIkVWaQkCRJlRkkJElS\nZf8fJizUt/VvYvkAAAAASUVORK5CYII=\n",
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"max_y = float(y[-1])\n",
"norm_y = [item/max_y for item in y]\n",
"plt.plot(x, norm_y)\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"0.9011726918107226"
]
},
"execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"norm_y[200]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Validate labels"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/html": [
"\n",
"
\n",
" \n",
" \n",
" | \n",
" id | \n",
" labels | \n",
"
\n",
" \n",
" \n",
" \n",
" | 0 | \n",
" --h7o0bAOpk | \n",
" 0 2 581 111 55 317 | \n",
"
\n",
" \n",
" | 1 | \n",
" --O5HgNtvSc | \n",
" 208 1 3073 1294 1073 | \n",
"
\n",
" \n",
" | 2 | \n",
" --q4Vg_nqEw | \n",
" 0 656 2 19 2677 | \n",
"
\n",
" \n",
" | 3 | \n",
" --wpV2HK8zQ | \n",
" 676 28 | \n",
"
\n",
" \n",
" | 4 | \n",
" --4eRn5OaI8 | \n",
" 3105 | \n",
"
\n",
" \n",
" | 5 | \n",
" ---FI9JLZRg | \n",
" 0 2944 | \n",
"
\n",
" \n",
" | 6 | \n",
" --vnrH7mF1k | \n",
" 1 484 345 | \n",
"
\n",
" \n",
" | 7 | \n",
" --7Z6cyWKmk | \n",
" 361 3 | \n",
"
\n",
" \n",
" | 8 | \n",
" --JGGtGgoqI | \n",
" 0 51 373 | \n",
"
\n",
" \n",
" | 9 | \n",
" --jKa62hkw0 | \n",
" 5 | \n",
"
\n",
" \n",
" | 10 | \n",
" --Yiihx8DQM | \n",
" 1 659 4 1828 519 | \n",
"
\n",
" \n",
" | 11 | \n",
" --AeBuy3cNY | \n",
" 1 4 94 575 | \n",
"
\n",
" \n",
" | 12 | \n",
" --hqmmJ2fqw | \n",
" 298 37 6 15 | \n",
"
\n",
" \n",
" | 13 | \n",
" --_Ch7tD8xg | \n",
" 200 232 126 | \n",
"
\n",
" \n",
" | 14 | \n",
" --q5_kUt7Qo | \n",
" 624 3 5 | \n",
"
\n",
" \n",
" | 15 | \n",
" --jHGbHxyOo | \n",
" 18 83 108 | \n",
"
\n",
" \n",
" | 16 | \n",
" --s_zBYzhgk | \n",
" 0 8 76 | \n",
"
\n",
" \n",
" | 17 | \n",
" --uMOMKVso4 | \n",
" 0 599 | \n",
"
\n",
" \n",
" | 18 | \n",
" --_fsPVHB6I | \n",
" 8 | \n",
"
\n",
" \n",
" | 19 | \n",
" --nokr3akJM | \n",
" 593 | \n",
"
\n",
" \n",
" | 20 | \n",
" --b5lu-49ik | \n",
" 0 2 1220 | \n",
"
\n",
" \n",
" | 21 | \n",
" --P7RAx3lrw | \n",
" 289 10 80 18 404 438 247 | \n",
"
\n",
" \n",
" | 22 | \n",
" --fK3UFck7c | \n",
" 1 1107 308 1715 | \n",
"
\n",
" \n",
" | 23 | \n",
" --W1zGEL5gU | \n",
" 3 | \n",
"
\n",
" \n",
" | 24 | \n",
" --mgF8UcvTU | \n",
" 68 2703 | \n",
"
\n",
" \n",
" | 25 | \n",
" --tvS5H063k | \n",
" 0 3649 2 34 199 | \n",
"
\n",
" \n",
" | 26 | \n",
" --2B4d4lQhs | \n",
" 104 36 53 | \n",
"
\n",
" \n",
" | 27 | \n",
" --QhOJxZyjc | \n",
" 46 13 38 7 | \n",
"
\n",
" \n",
" | 28 | \n",
" --nec1RQWNY | \n",
" 2062 | \n",
"
\n",
" \n",
" | 29 | \n",
" --0gptpMDmY | \n",
" 66 | \n",
"
\n",
" \n",
" | ... | \n",
" ... | \n",
" ... | \n",
"
\n",
" \n",
" | 1401798 | \n",
" zzm2O40aPJc | \n",
" 8 54 79 | \n",
"
\n",
" \n",
" | 1401799 | \n",
" zzpLddu603Q | \n",
" 101 23 1009 2834 1207 29 | \n",
"
\n",
" \n",
" | 1401800 | \n",
" zz5irKkXBdM | \n",
" 1 1025 42 11 4349 114 147 1076 87 2282 | \n",
"
\n",
" \n",
" | 1401801 | \n",
" zzJpVCsXhZQ | \n",
" 148 61 125 | \n",
"
\n",
" \n",
" | 1401802 | \n",
" zzuDlhu9sLQ | \n",
" 56 890 28 1215 | \n",
"
\n",
" \n",
" | 1401803 | \n",
" zzBMXp4D3m4 | \n",
" 1 4 135 1169 41 11 17 | \n",
"
\n",
" \n",
" | 1401804 | \n",
" zzL_IeqdcLA | \n",
" 1 13 | \n",
"
\n",
" \n",
" | 1401805 | \n",
" zzQZvZNaf3E | \n",
" 1742 | \n",
"
\n",
" \n",
" | 1401806 | \n",
" zzwpk_6QnTA | \n",
" 3 | \n",
"
\n",
" \n",
" | 1401807 | \n",
" zzmxIhDgd8o | \n",
" 4268 13 726 7 | \n",
"
\n",
" \n",
" | 1401808 | \n",
" zzZT4OM26ls | \n",
" 110 14 7 | \n",
"
\n",
" \n",
" | 1401809 | \n",
" zzsBXUxAR0g | \n",
" 512 228 1063 2634 525 47 | \n",
"
\n",
" \n",
" | 1401810 | \n",
" zzw1p2KHi1A | \n",
" 429 23 | \n",
"
\n",
" \n",
" | 1401811 | \n",
" zz5LQRMqDTE | \n",
" 48 10 636 | \n",
"
\n",
" \n",
" | 1401812 | \n",
" zzPUcw7_RBg | \n",
" 291 235 45 436 56 28 | \n",
"
\n",
" \n",
" | 1401813 | \n",
" zztZPYsbrvo | \n",
" 0 2 19 506 1382 | \n",
"
\n",
" \n",
" | 1401814 | \n",
" zzyuSRyfNcw | \n",
" 1 42 308 605 87 | \n",
"
\n",
" \n",
" | 1401815 | \n",
" zzCz1UtVRVg | \n",
" 160 1 129 82 50 149 92 | \n",
"
\n",
" \n",
" | 1401816 | \n",
" zzW5krTlh_U | \n",
" 32 1223 232 1322 295 12 | \n",
"
\n",
" \n",
" | 1401817 | \n",
" zzAzChc2wBw | \n",
" 0 8 21 54 79 | \n",
"
\n",
" \n",
" | 1401818 | \n",
" zz8Ah6Yo7Gw | \n",
" 1376 10 2479 440 1591 280 217 | \n",
"
\n",
" \n",
" | 1401819 | \n",
" zzG3TcgGYNA | \n",
" 89 387 269 4422 855 | \n",
"
\n",
" \n",
" | 1401820 | \n",
" zzRUfWjGz9M | \n",
" 0 171 159 | \n",
"
\n",
" \n",
" | 1401821 | \n",
" zzVTEl2QJBw | \n",
" 0 2 34 118 3797 | \n",
"
\n",
" \n",
" | 1401822 | \n",
" zzFGqSCp3t4 | \n",
" 83 108 221 | \n",
"
\n",
" \n",
" | 1401823 | \n",
" zzpJaaj0FIg | \n",
" 3 | \n",
"
\n",
" \n",
" | 1401824 | \n",
" zz-3UjTglRY | \n",
" 20 | \n",
"
\n",
" \n",
" | 1401825 | \n",
" zz9LRC0P8Os | \n",
" 0 25 2 23 | \n",
"
\n",
" \n",
" | 1401826 | \n",
" zz4DceTfj-c | \n",
" 1946 15 | \n",
"
\n",
" \n",
" | 1401827 | \n",
" zzvd6yEiOmk | \n",
" 0 1 2 388 50 | \n",
"
\n",
" \n",
"
\n",
"
1401828 rows × 2 columns
\n",
"
"
],
"text/plain": [
" id labels\n",
"0 --h7o0bAOpk 0 2 581 111 55 317\n",
"1 --O5HgNtvSc 208 1 3073 1294 1073\n",
"2 --q4Vg_nqEw 0 656 2 19 2677\n",
"3 --wpV2HK8zQ 676 28\n",
"4 --4eRn5OaI8 3105\n",
"5 ---FI9JLZRg 0 2944\n",
"6 --vnrH7mF1k 1 484 345\n",
"7 --7Z6cyWKmk 361 3\n",
"8 --JGGtGgoqI 0 51 373\n",
"9 --jKa62hkw0 5\n",
"10 --Yiihx8DQM 1 659 4 1828 519\n",
"11 --AeBuy3cNY 1 4 94 575\n",
"12 --hqmmJ2fqw 298 37 6 15\n",
"13 --_Ch7tD8xg 200 232 126\n",
"14 --q5_kUt7Qo 624 3 5\n",
"15 --jHGbHxyOo 18 83 108\n",
"16 --s_zBYzhgk 0 8 76\n",
"17 --uMOMKVso4 0 599\n",
"18 --_fsPVHB6I 8\n",
"19 --nokr3akJM 593\n",
"20 --b5lu-49ik 0 2 1220\n",
"21 --P7RAx3lrw 289 10 80 18 404 438 247\n",
"22 --fK3UFck7c 1 1107 308 1715\n",
"23 --W1zGEL5gU 3\n",
"24 --mgF8UcvTU 68 2703\n",
"25 --tvS5H063k 0 3649 2 34 199\n",
"26 --2B4d4lQhs 104 36 53\n",
"27 --QhOJxZyjc 46 13 38 7\n",
"28 --nec1RQWNY 2062\n",
"29 --0gptpMDmY 66\n",
"... ... ...\n",
"1401798 zzm2O40aPJc 8 54 79\n",
"1401799 zzpLddu603Q 101 23 1009 2834 1207 29\n",
"1401800 zz5irKkXBdM 1 1025 42 11 4349 114 147 1076 87 2282\n",
"1401801 zzJpVCsXhZQ 148 61 125\n",
"1401802 zzuDlhu9sLQ 56 890 28 1215\n",
"1401803 zzBMXp4D3m4 1 4 135 1169 41 11 17\n",
"1401804 zzL_IeqdcLA 1 13\n",
"1401805 zzQZvZNaf3E 1742\n",
"1401806 zzwpk_6QnTA 3\n",
"1401807 zzmxIhDgd8o 4268 13 726 7\n",
"1401808 zzZT4OM26ls 110 14 7\n",
"1401809 zzsBXUxAR0g 512 228 1063 2634 525 47\n",
"1401810 zzw1p2KHi1A 429 23\n",
"1401811 zz5LQRMqDTE 48 10 636\n",
"1401812 zzPUcw7_RBg 291 235 45 436 56 28\n",
"1401813 zztZPYsbrvo 0 2 19 506 1382\n",
"1401814 zzyuSRyfNcw 1 42 308 605 87\n",
"1401815 zzCz1UtVRVg 160 1 129 82 50 149 92\n",
"1401816 zzW5krTlh_U 32 1223 232 1322 295 12\n",
"1401817 zzAzChc2wBw 0 8 21 54 79\n",
"1401818 zz8Ah6Yo7Gw 1376 10 2479 440 1591 280 217\n",
"1401819 zzG3TcgGYNA 89 387 269 4422 855\n",
"1401820 zzRUfWjGz9M 0 171 159\n",
"1401821 zzVTEl2QJBw 0 2 34 118 3797\n",
"1401822 zzFGqSCp3t4 83 108 221\n",
"1401823 zzpJaaj0FIg 3\n",
"1401824 zz-3UjTglRY 20\n",
"1401825 zz9LRC0P8Os 0 25 2 23\n",
"1401826 zz4DceTfj-c 1946 15\n",
"1401827 zzvd6yEiOmk 0 1 2 388 50\n",
"\n",
"[1401828 rows x 2 columns]"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"validate_labels"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0-Games / 2-Video game / 55-Video game console / 111-PlayStation 3 / 317-PlayStation / 581-PlayStation / \n",
"1-Vehicle / 208-Wood / 1073-Chainsaw / 1294-Logging / 3073-Log splitter / \n",
"0-Games / 2-Video game / 19-PC game / 656-Metal Gear / 2677-Metal Gear Online / \n",
"28-Fashion / 676-Handbag / \n",
"3105-Numerical digit / \n",
"0-Games / 2944-La Tale / \n",
"1-Vehicle / 345-Loudspeaker / 484-Subwoofer / \n",
"3-Concert / 361-Flamenco / \n",
"0-Games / 51-Strategy video game / 373-Clash of Clans / \n",
"5-Dance / \n",
"1-Vehicle / 4-Car / 519-Recreational vehicle / 659-Van / 1828-Campervan / \n",
"1-Vehicle / 4-Car / 94-Dashcam / 575-Traffic / \n",
"6-Animation / 15-Cartoon / 37-Gadget / 298-Television advertisement / \n",
"126-The Walt Disney Company / 200-Doll / 232-Cake / \n",
"3-Concert / 5-Dance / 624-Number / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / \n",
"0-Games / 8-Football / 76-Ball / \n",
"0-Games / 599-Dofus / \n",
"8-Football / \n",
"593-Moon / \n",
"0-Games / 2-Video game / 1220-New Super Mario Bros / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 247-Stallion / 289-Dressage / 404-Livestock / 438-Mare / \n",
"1-Vehicle / 308-Ship / 1107-United States Navy / 1715-Aircraft carrier / \n",
"3-Concert / \n",
"68-Lighting / 2703-Light fixture / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 199-Wii / 3649-Metroid Prime / \n",
"36-Piano / 53-Keyboard / 104-Pianist / \n",
"7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"2062-Kidney / \n",
"66-Bollywood / \n",
"273-News program / \n",
"49-School / \n",
"10-Animal / 18-Outdoor recreation / 42-Fishing / 91-Fish / 3292-Rock fishing / \n",
"56-Hair / 106-Hairstyle / 871-Hair coloring / 1269-Human hair color / \n",
"13-Musical ensemble / 31-Disc jockey / \n",
"5-Dance / 399-Kickboxing / \n",
"9-Music video / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"3-Concert / 14-Guitar / \n",
"45-Cosmetics / 282-Skin / 291-Rouge / \n",
"9-Music video / \n",
"45-Cosmetics / 193-Eye liner / 195-Mascara / 218-Eyelash / 235-Lipstick / 291-Rouge / 436-Concealer / \n",
"28-Fashion / 276-Room / \n",
"28-Fashion / 205-Newscaster / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"0-Games / 2-Video game / 33-Weapon / 948-Far Cry / 3618-Far Cry / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 135-Drifting / 201-Grand Theft Auto: San Andreas / \n",
"1-Vehicle / 4-Car / 297-Grand Theft Auto IV / \n",
"242-Paper / 1642-Cigarette / \n",
"0-Games / 2-Video game / 1288-Marvel vs. Capcom / 3081-Marvel vs. Capcom 2: New Age of Heroes / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 272-iPod / 331-iPod touch / \n",
"5-Dance / 769-Red Bull / \n",
"359-Harry Potter (Literary Series) / \n",
"0-Games / 2-Video game / 25-Toy / 140-LEGO / 199-Wii / 443-Wii U / 1079-Lego City / 4016-Lego City Undercover / \n",
"28-Fashion / 45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 235-Lipstick / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"164-Beach / 369-Resort / \n",
"0-Games / 8-Football / 54-Highlight film / 76-Ball / \n",
"12-Food / 170-Home improvement / 246-Furniture / 303-Paint / 1272-Cabinetry / \n",
"45-Cosmetics / 56-Hair / 235-Lipstick / \n",
"1880-Marinera / \n",
"6-Animation / 15-Cartoon / \n",
"36-Piano / 53-Keyboard / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 2-Video game / 241-Fighting game / 764-The King of Fighters / 3818-The King of Fighters '98 / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 380-Assassin's Creed / \n",
"1182-Softball / \n",
"3-Concert / 5-Dance / \n",
"3-Concert / \n",
"75-Wrestling / \n",
"5-Dance / 16-Performance art / 49-School / 66-Bollywood / \n",
"959-Metal detector / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 1040-Street racing / \n",
"67-Cycling / 69-Bicycle / \n",
"67-Cycling / \n",
"230-Brass instrument / 427-Trumpet / \n",
"28-Fashion / 45-Cosmetics / 1352-Airbrush / \n",
"0-Games / 264-Star Wars / 714-Jedi / \n",
"2-Video game / 6-Animation / 199-Wii / 1004-Samurai / \n",
"10-Animal / 71-Dog / \n",
"1-Vehicle / 27-Motorcycle / 245-BMW / 2763-BMW R1200GS / 2975-BMW Motorrad / \n",
"12-Food / 86-Plant / \n",
"5-Dance / 99-Christmas / 558-Santa Claus / \n",
"20-Trailer / 643-Hero / 1632-Villain / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"28-Fashion / 45-Cosmetics / 56-Hair / 173-Eye shadow / 195-Mascara / 218-Eyelash / \n",
"5-Dance / 1139-Belle / 1265-Kyle Kingson / \n",
"5-Dance / 16-Performance art / \n",
"1-Vehicle / 4-Car / \n",
"227-Water / 640-Bottle / 1058-Volcano / \n",
"56-Hair / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"455-Unidentified flying object / \n",
"5-Dance / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 77-Arena / 81-Athlete / 1247-Penalty kick / \n",
"60-Basketball / \n",
"3-Concert / 7-Musician / \n",
"12-Food / \n",
"1782-Supermarket / \n",
"15-Cartoon / 467-Chipmunk / \n",
"20-Trailer / \n",
"0-Games / 1-Vehicle / 33-Weapon / 51-Strategy video game / 271-Tank / 409-World of Tanks / \n",
"1-Vehicle / 18-Outdoor recreation / 147-River / 481-Kayak / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 220-Mountain bike / \n",
"1-Vehicle / 4-Car / 114-Engine / \n",
"66-Bollywood / \n",
"2-Video game / 47-Personal computer / 131-Computer / 309-Computer hardware / 700-Hard disk drive / \n",
"0-Games / 2-Video game / 51-Strategy video game / 165-League of Legends / \n",
"10-Animal / 12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 91-Fish / 109-Cooking show / 239-Roasting / 391-Chicken meat / 454-Sauce / 893-Fish as food / 2125-Fillet / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"3-Concert / 16-Performance art / 68-Lighting / \n",
"0-Games / 2-Video game / 43-Call of Duty / 55-Video game console / 111-PlayStation 3 / 216-Call of Duty: Modern Warfare 3 / \n",
"93-Comedy / \n",
"3-Concert / \n",
"257-Counter-Strike / 1226-Metalworking / \n",
"36-Piano / 53-Keyboard / 397-Organ / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / \n",
"3-Concert / 16-Performance art / \n",
"1-Vehicle / 4-Car / 683-Renault / 4672-Renault 5 / \n",
"5-Dance / 78-Wedding / 153-Dress / \n",
"10-Animal / 22-Nature / 42-Fishing / 718-Duck / 814-Pond / \n",
"12-Food / 52-Dish / 58-Cuisine / 86-Plant / 500-Rice / 1438-Street food / 2037-Straw / \n",
"0-Games / 1644-ArcheAge / 2840-Engraving / \n",
"9-Music video / \n",
"12-Food / 49-School / \n",
"0-Games / 2-Video game / 638-Gears of War / 773-Gears of War / \n",
"0-Games / 35-Minecraft / \n",
"20-Trailer / \n",
"8-Football / 54-Highlight film / 65-Kick / \n",
"6-Animation / 123-Naruto / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"444-Family / \n",
"9-Music video / \n",
"66-Bollywood / \n",
"9-Music video / \n",
"18-Outdoor recreation / 22-Nature / 283-Rock / 347-Climbing / 351-Jumping / 548-Parachuting / 669-Extreme sport / \n",
"7-Musician / 14-Guitar / 24-String instrument / 30-Drummer / 39-Drums / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 19-PC game / 34-Action-adventure game / 118-Grand Theft Auto V / 1275-Watch Dogs / \n",
"78-Wedding / 121-Photography / 190-Bride / \n",
"0-Games / 2-Video game / 47-Personal computer / 345-Loudspeaker / 484-Subwoofer / \n",
"25-Toy / 269-Comic book / \n",
"9-Music video / \n",
"164-Beach / 502-Sky / 573-Paragliding / 1561-Paramotor / \n",
"3-Concert / \n",
"3-Concert / \n",
"12-Food / 26-Cooking / 32-Recipe / 227-Water / \n",
"102-Building / 885-Bell / \n",
"238-Nail / 304-Nail polish / 1498-Key / \n",
"6-Animation / 15-Cartoon / 185-Dragon Ball / 244-Goku / 418-Gohan / 876-Freeza / \n",
"6-Animation / 183-Manga / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 77-Arena / 81-Athlete / \n",
"14-Guitar / \n",
"12-Food / 26-Cooking / 32-Recipe / 109-Cooking show / 236-Baking / 442-Bread / \n",
"36-Piano / 53-Keyboard / 379-Touhou Project / 3528-Imperishable Night / \n",
"1-Vehicle / 11-Motorsport / 124-Tractor / 156-Agriculture / 174-Farm / 252-Tractor pulling / \n",
"3-Concert / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"10-Animal / 22-Nature / 132-Bird / 137-Tree / \n",
"18-Outdoor recreation / 81-Athlete / 287-Human swimming / 2605-Medley swimming / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 30-Drummer / 39-Drums / 44-Drums / \n",
"0-Games / 2-Video game / 43-Call of Duty / 138-Call of Duty: Black Ops II / \n",
"13-Musical ensemble / \n",
"14-Guitar / \n",
"8-Football / \n",
"0-Games / 1-Vehicle / 2-Video game / 34-Action-adventure game / 102-Building / 118-Grand Theft Auto V / \n",
"8-Football / \n",
"18-Outdoor recreation / 147-River / 794-Bridge / \n",
"116-Pokémon / \n",
"0-Games / 2-Video game / 14-Guitar / 445-Guitar Hero / 3558-Guitar Hero: Metallica / \n",
"9-Music video / \n",
"8-Football / \n",
"1678-Foam / \n",
"1-Vehicle / 27-Motorcycle / 207-Exhaust system / 1380-Kawasaki motorcycles / 2404-Kawasaki Ninja / \n",
"20-Trailer / 390-Model / \n",
"102-Building / 113-House / \n",
"1-Vehicle / 4-Car / 84-Snow / 120-Winter / \n",
"3-Concert / \n",
"3-Concert / 5-Dance / 483-Costume / \n",
"3169-Diva / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 86-Plant / 213-Forest / 217-Hunting / 280-Wildlife / 706-Deer / 709-Bull / 2804-Elk / \n",
"616-The Bible / \n",
"13-Musical ensemble / 28-Fashion / 153-Dress / \n",
"5-Dance / 16-Performance art / 1422-Statue / \n",
"1-Vehicle / 50-Aircraft / 139-Airport / 157-Jet aircraft / 175-Airliner / 188-Airline / 1340-Airbus A380 / 2790-British Airways / \n",
"131-Computer / 141-Microsoft Windows / \n",
"47-Personal computer / 131-Computer / 414-Computer monitor / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / 691-Cargo / \n",
"10-Animal / 91-Fish / 258-Aquarium / 366-Dragon / \n",
"410-Teacher / \n",
"96-Soldier / \n",
"6-Animation / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 1169-Hot Wheels / \n",
"3-Concert / 7-Musician / 14-Guitar / 49-School / 98-Festival / \n",
"0-Games / 51-Strategy video game / 373-Clash of Clans / \n",
"10-Animal / 48-Pet / 71-Dog / 881-German Shepherd / \n",
"3-Concert / 7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / 150-Violin / 726-Clarinet / \n",
"3-Concert / 6-Animation / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 92-Radio-controlled model / 149-Model aircraft / 157-Jet aircraft / 160-Radio-controlled aircraft / \n",
"7-Musician / 14-Guitar / 24-String instrument / 219-Accordion / 2185-PRS Guitars / \n",
"2-Video game / 55-Video game console / 118-Grand Theft Auto V / \n",
"223-Weather / 225-Lake / \n",
"1-Vehicle / 4-Car / 40-Road / 94-Dashcam / 575-Traffic / 663-Street / 1960-Traffic light / \n",
"7-Musician / 14-Guitar / \n",
"6-Animation / 15-Cartoon / \n",
"744-Window / 4448-Window cleaner / \n",
"1-Vehicle / 4-Car / 509-Classic car / 1321-Škoda Auto / \n",
"0-Games / 2-Video game / 237-Final Fantasy / 1131-Final Fantasy XIII / 2506-Final Fantasy XIII-2 / \n",
"5-Dance / 7-Musician / 13-Musical ensemble / 49-School / 203-Marching band / 3669-Oceanic dolphin / \n",
"7-Musician / 9-Music video / \n",
"1-Vehicle / 11-Motorsport / 1025-Personal water craft / 1336-Jet Ski / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"31-Disc jockey / \n",
"1-Vehicle / 4-Car / 4115-Nissan JUKE / \n",
"3-Concert / 9-Music video / 178-Samsung Galaxy / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 192-Battlefield / 464-First-person Shooter / 1267-Battlefield Hardline / \n",
"5-Dance / \n",
"67-Cycling / 69-Bicycle / 520-Bicycle frame / \n",
"0-Games / 530-Cue sports / 568-Pool / 994-Cue stick / 1334-Eight-ball / 1364-Nine-ball / \n",
"3-Concert / \n",
"5-Dance / \n",
"6-Animation / 15-Cartoon / \n",
"444-Family / \n",
"223-Weather / \n",
"38-Orchestra / \n",
"3-Concert / 5-Dance / 30-Drummer / \n",
"36-Piano / 53-Keyboard / \n",
"6-Animation / 801-Inuyasha / \n",
"28-Fashion / 45-Cosmetics / 466-Gift / 4258-Raiden / \n",
"6-Animation / 13-Musical ensemble / \n",
"8-Football / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"0-Games / 326-Metin2 / \n",
"148-Painting / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 378-Drag racing / 473-Ford Mustang / \n",
"1-Vehicle / 4-Car / 234-Ford / 473-Ford Mustang / 918-Brake / \n",
"1-Vehicle / 4-Car / 389-Restaurant / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"1151-Temple / \n",
"1449-Rubber band / \n",
"1-Vehicle / 117-Boxing / \n",
"20-Trailer / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 161-Train station / 224-Rapid transit / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 918-Brake / \n",
"938-Steel / 3515-Gas tungsten arc welding / \n",
"1939-Logic Pro / \n",
"1-Vehicle / 87-Boat / 308-Ship / 2007-Intermodal container / \n",
"0-Games / 326-Metin2 / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 105-Motorcycling / \n",
"9-Music video / \n",
"115-Ballet / 2339-Swan / \n",
"5-Dance / 16-Performance art / \n",
"0-Games / 2-Video game / 19-PC game / 192-Battlefield / 266-Helicopter / 374-Battlefield 3 / 3513-Split screen / \n",
"0-Games / 2-Video game / 55-Video game console / 116-Pokémon / 1118-Kirby / 1133-Kirby / 1446-Game Boy / 1637-GameCube / \n",
"113-House / 170-Home improvement / \n",
"1-Vehicle / 87-Boat / 308-Ship / 3163-Tugboat / \n",
"1-Vehicle / 4-Car / \n",
"5-Dance / 68-Lighting / 133-Nightclub / \n",
"5-Dance / 410-Teacher / \n",
"0-Games / 2-Video game / 229-Halo / 644-Halo: Reach / \n",
"246-Furniture / 276-Room / 416-Apartment / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / \n",
"248-Hotel / 369-Resort / \n",
"3-Concert / 68-Lighting / \n",
"0-Games / 60-Basketball / \n",
"208-Wood / 261-Tool / 315-Knife / 400-Woodturning / 2017-Sharpening / 4309-Plane / \n",
"0-Games / 2-Video game / 20-Trailer / 192-Battlefield / 374-Battlefield 3 / 383-Battlefield 4 / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"0-Games / 75-Wrestling / 977-WWE '13 / \n",
"0-Games / 2-Video game / 6-Animation / 506-Garry's Mod / \n",
"1-Vehicle / 27-Motorcycle / 105-Motorcycling / 114-Engine / 207-Exhaust system / \n",
"1-Vehicle / 59-Winter sport / 84-Snow / 120-Winter / 177-Skiing / 281-Snowboarding / 305-Ski / 501-Snowboard / \n",
"315-Knife / \n",
"1-Vehicle / 4-Car / 114-Engine / 2377-Cylinder / \n",
"0-Games / 271-Tank / 409-World of Tanks / \n",
"15-Cartoon / 750-Sailor Moon / \n",
"144-Amusement park / 3794-Legoland / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 18-Outdoor recreation / 87-Boat / 147-River / \n",
"0-Games / 8-Football / 77-Arena / 90-Sports game / 180-FIFA 15 / \n",
"0-Games / 2-Video game / 90-Sports game / 180-FIFA 15 / 480-FIFA 13 / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"0-Games / 75-Wrestling / \n",
"444-Family / \n",
"10-Animal / 48-Pet / 166-Cat / 346-Kitten / \n",
"3-Concert / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 39-Drums / \n",
"85-Combat / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"6-Animation / 15-Cartoon / 61-Art / 97-Drawing / 4248-Spacetoon / \n",
"1-Vehicle / 4-Car / 114-Engine / 945-MINI Cooper / 1221-MINI / \n",
"10-Animal / 48-Pet / 71-Dog / 132-Bird / 598-Parrot / \n",
"512-Printing / 525-Printer / 551-British Broadcasting Corporation / \n",
"5-Dance / 16-Performance art / \n",
"1-Vehicle / 610-Snowmobile / \n",
"2-Video game / 159-World of Warcraft / \n",
"10-Animal / 523-Cattle / 1457-Herd / \n",
"12-Food / 232-Cake / 1175-Soap / \n",
"0-Games / 2-Video game / 19-PC game / 686-Diablo III / 872-Diablo / \n",
"1-Vehicle / 4-Car / 84-Snow / 158-Tire / 167-Four-wheel drive / 198-Off-road vehicle / 360-Jeep / 1205-Jeep Wrangler / \n",
"361-Flamenco / \n",
"6-Animation / 10-Animal / 15-Cartoon / 3332-Hippopotamus / \n",
"0-Games / 51-Strategy video game / 174-Farm / 373-Clash of Clans / \n",
"18-Outdoor recreation / 84-Snow / 120-Winter / 137-Tree / 347-Climbing / 395-Camping / 1203-Backpacking / 3065-Hammock / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / \n",
"45-Cosmetics / 56-Hair / \n",
"9-Music video / \n",
"8-Football / \n",
"12-Food / \n",
"25-Toy / 140-LEGO / 264-Star Wars / \n",
"1105-Fingerboard / \n",
"1-Vehicle / 62-Train / 64-Transport / 94-Dashcam / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 691-Cargo / \n",
"36-Piano / 53-Keyboard / \n",
"110-Album / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 274-Meat / 391-Chicken meat / 572-Indian cuisine / 589-Grilling / 620-Barbecue / 663-Street / 1438-Street food / \n",
"12-Food / 26-Cooking / 32-Recipe / 226-Dessert / 232-Cake / 236-Baking / 295-Chocolate / 437-Cream / 560-Sugar / 578-Cookie / 684-Ice cream / 759-Icing / 795-Cupcake / 966-Sandwich / 1408-Snack / 1541-Buttercream / 4319-Oreo / \n",
"122-Weight training / 516-Headphones / 529-Squat / \n",
"10-Animal / 48-Pet / 71-Dog / 1278-Labrador Retriever / \n",
"8-Football / \n",
"6-Animation / 15-Cartoon / 637-Five Nights at Freddy's / \n",
"0-Games / 2-Video game / 51-Strategy video game / 1570-Command & Conquer / 4526-Command & Conquer: Generals / \n",
"0-Games / 2-Video game / 1027-Terraria / 1130-Armour / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"8-Football / 21-Stadium / \n",
"9-Music video / \n",
"0-Games / 51-Strategy video game / 165-League of Legends / \n",
"1066-Aikido / 2531-Dojo / \n",
"0-Games / 8-Football / 79-American football / \n",
"5-Dance / 133-Nightclub / 342-Windows Media Video / \n",
"93-Comedy / 355-Comedian / \n",
"43-Call of Duty / 302-Call of Duty 4: Modern Warfare / \n",
"14-Guitar / \n",
"6-Animation / 15-Cartoon / 2784-Sponge / \n",
"3-Concert / 7-Musician / \n",
"5-Dance / 95-Talent show / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 441-The Elder Scrolls / 1919-The Elder Scrolls Online / 3569-Scrolls / \n",
"1-Vehicle / 4-Car / 6-Animation / 2769-Decal / \n",
"0-Games / 2-Video game / 2491-Spec Ops: The Line / \n",
"30-Drummer / 36-Piano / 53-Keyboard / 107-Musical keyboard / 397-Organ / \n",
"88-Machine / 3515-Gas tungsten arc welding / 3929-Gas metal arc welding / \n",
"10-Animal / \n",
"609-Tattoo / \n",
"43-Call of Duty / 194-Call of Duty: Modern Warfare 2 / \n",
"0-Games / 2-Video game / 366-Dragon / 412-Monster / \n",
"12-Food / 26-Cooking / 32-Recipe / 232-Cake / 485-Dough / 560-Sugar / 578-Cookie / \n",
"110-Album / \n",
"3-Concert / \n",
"3-Concert / 14-Guitar / 16-Performance art / \n",
"117-Boxing / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 40-Road / 57-Race track / 105-Motorcycling / 2133-Road racing / \n",
"3-Concert / 30-Drummer / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / 216-Call of Duty: Modern Warfare 3 / \n",
"3-Concert / 7-Musician / 24-String instrument / 39-Drums / \n",
"75-Wrestling / \n",
"72-iPhone / 189-iPad / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / 183-Manga / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"9-Music video / 92-Radio-controlled model / 149-Model aircraft / 155-Camera / 160-Radio-controlled aircraft / 278-GoPro / 393-Unmanned aerial vehicle / 604-Quadcopter / \n",
"66-Bollywood / 355-Comedian / \n",
"12-Food / 26-Cooking / 52-Dish / 58-Cuisine / 121-Photography / 301-Meal / 2040-Korean food / \n",
"60-Basketball / 1399-Streetball / \n",
"10-Animal / 22-Nature / 80-Horse / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 277-Sport utility vehicle / 1391-Kia / 1415-Kia Motors / 4398-Kia Sorento / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 30-Drummer / 44-Drums / 1533-Lead guitar / \n",
"248-Hotel / 369-Resort / \n",
"1524-Zee Bangla / \n",
"10-Animal / 22-Nature / 71-Dog / 217-Hunting / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / 806-Chopper / \n",
"0-Games / 2-Video game / 12-Food / 713-Egg / 1047-Easter egg / 1069-Easter egg / \n",
"6-Animation / 15-Cartoon / 849-Tales / 2195-Tales of Symphonia / \n",
"512-Printing / \n",
"1-Vehicle / 40-Road / 938-Steel / \n",
"6-Animation / 15-Cartoon / 2403-Crayon Shin-chan / \n",
"22-Nature / 86-Plant / 102-Building / 657-Autumn / \n",
"102-Building / 113-House / 170-Home improvement / 179-Kitchen / 205-Newscaster / 276-Room / 720-Wine / 2055-Mansion / \n",
"1-Vehicle / 4-Car / 84-Snow / 120-Winter / 1768-Snow blower / \n",
"5-Dance / 16-Performance art / 430-Ballroom dance / 678-Latin dance / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 176-PlayStation 4 / 656-Metal Gear / 2212-Metal Gear Solid V: The Phantom Pain / 4027-Metal Gear Solid V: Ground Zeroes / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 16-Performance art / 24-String instrument / \n",
"0-Games / 362-PlayStation Portable / \n",
"0-Games / 2-Video game / 1138-Tower of Saviors / \n",
"12-Food / 915-Town / \n",
"199-Wii / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 1031-Fisheye lens / \n",
"1-Vehicle / 4-Car / 87-Boat / 114-Engine / \n",
"7-Musician / 14-Guitar / 24-String instrument / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 70-Driving / 1056-Porsche 911 / \n",
"356-Star / \n",
"3-Concert / 5-Dance / \n",
"9-Music video / \n",
"9-Music video / 22-Nature / 267-Mountain / 283-Rock / \n",
"22-Nature / 86-Plant / 121-Photography / \n",
"0-Games / 2-Video game / 271-Tank / 1167-Tanki Online / \n",
"10-Animal / 12-Food / 91-Fish / 1018-Salmon / 1227-Sushi / \n",
"7-Musician / 46-Choir / \n",
"269-Comic book / 357-DVD / 950-Blu-ray disc / \n",
"548-Parachuting / 573-Paragliding / 765-Parachute / \n",
"0-Games / 2-Video game / 15-Cartoon / 89-Comics / 185-Dragon Ball / 244-Goku / \n",
"0-Games / 2-Video game / 35-Minecraft / 47-Personal computer / 145-Tablet computer / \n",
"0-Games / 2-Video game / 928-Borderlands 2 / 2613-Borderlands: The Pre-Sequel! / \n",
"15-Cartoon / 183-Manga / \n",
"708-Mermaid / 1219-Ariel / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / \n",
"59-Winter sport / 112-Ice skating / 142-Hockey / \n",
"20-Trailer / 4211-Walter White / \n",
"0-Games / 2-Video game / 8-Football / 79-American football / 90-Sports game / 561-Madden NFL / \n",
"83-Skateboarding / 108-Skateboard / 556-Skateboarding trick / 1315-Kickflip / 1454-Stairs / 4097-Nollie / \n",
"6-Animation / 25-Toy / 701-Beyblade / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 169-Rallying / \n",
"1-Vehicle / 22-Nature / 124-Tractor / 156-Agriculture / 249-Heavy equipment / 617-Maize / \n",
"3-Concert / 5-Dance / 16-Performance art / 68-Lighting / \n",
"7-Musician / 36-Piano / \n",
"3-Concert / 46-Choir / \n",
"6-Animation / \n",
"6-Animation / 15-Cartoon / 1211-Transformice / \n",
"0-Games / 8-Football / 77-Arena / 2985-Chunichi Dragons Doala / \n",
"269-Comic book / \n",
"75-Wrestling / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 2554-F.E.A.R. / \n",
"0-Games / 35-Minecraft / \n",
"54-Highlight film / 79-American football / \n",
"0-Games / 8-Football / 79-American football / 358-Rugby football / \n",
"22-Nature / 36-Piano / \n",
"13-Musical ensemble / 17-Racing / 203-Marching band / \n",
"49-School / 286-Bus / 1013-Walking / 2199-School bus / \n",
"5-Dance / 3902-Just Dance 3 / 4437-Just Dance 2 / \n",
"1-Vehicle / 11-Motorsport / 198-Off-road vehicle / 213-Forest / 327-Mud / 565-Mud bogging / \n",
"1-Vehicle / 221-Skatepark / 336-Scooter / \n",
"5-Dance / 78-Wedding / 2653-Wedding ring / \n",
"93-Comedy / \n",
"3-Concert / \n",
"31-Disc jockey / \n",
"350-Circus / \n",
"18-Outdoor recreation / 42-Fishing / 233-Ocean / 275-Diving / 381-Underwater diving / 2618-Lobster / \n",
"9-Music video / \n",
"75-Wrestling / 1582-WWE 2K14 / \n",
"5-Dance / \n",
"14-Guitar / \n",
"9-Music video / 66-Bollywood / \n",
"9-Music video / 14-Guitar / 488-Door / \n",
"6-Animation / 15-Cartoon / 971-Alphabet / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / 224-Rapid transit / 1547-Passenger / \n",
"0-Games / 739-Guild Wars / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 96-Soldier / 422-Call of Duty: Ghosts / 3522-ProSieben / \n",
"5-Dance / 66-Bollywood / \n",
"9-Music video / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"0-Games / 211-RuneScape / \n",
"110-Album / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 231-Supercar / 790-Nissan GT-R / 1301-Nissan Skyline / 2247-Nissan Skyline GT-R / \n",
"0-Games / 4467-Jade Dynasty / \n",
"3-Concert / \n",
"22-Nature / 657-Autumn / \n",
"1-Vehicle / 50-Aircraft / 129-Aviation / 139-Airport / 151-Landing / 187-Takeoff / 451-Cockpit / \n",
"10-Animal / 20-Trailer / 80-Horse / \n",
"3-Concert / 16-Performance art / \n",
"3-Concert / 5-Dance / 49-School / 776-Kindergarten / \n",
"3-Concert / 7-Musician / 14-Guitar / 98-Festival / 670-Banjo / \n",
"59-Winter sport / 112-Ice skating / 142-Hockey / \n",
"0-Games / 2-Video game / 586-Nintendo Entertainment System / \n",
"1-Vehicle / 4-Car / 17-Racing / 40-Road / 57-Race track / 94-Dashcam / 245-BMW / 254-Runway / 874-BMW M3 / 1607-BMW 3 Series (E36) / \n",
"5-Dance / 59-Winter sport / 112-Ice skating / 316-Figure skating / \n",
"3-Concert / 5-Dance / 31-Disc jockey / 133-Nightclub / 559-Night / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"24-String instrument / 150-Violin / 450-Viola / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 154-Wheel / 158-Tire / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"6-Animation / 15-Cartoon / 1771-The Pink Panther / 2520-Pink Panther / \n",
"89-Comics / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 105-Motorcycling / 172-Motocross / 363-Soil / 669-Extreme sport / \n",
"1-Vehicle / 4-Car / 10-Animal / 12-Food / 26-Cooking / 274-Meat / 723-Pig / 862-Pork / 1123-Domestic pig / 1557-Bacon / 1843-Ham / 2395-Ribs / \n",
"0-Games / 2-Video game / 6-Animation / 89-Comics / 123-Naruto / 285-Sasuke Uchiha / 591-Naruto: Ultimate Ninja / 1421-Naruto: Ultimate Ninja Storm / 2158-Naruto Shippuden: Ultimate Ninja Storm Generations / \n",
"45-Cosmetics / 56-Hair / 155-Camera / \n",
"1-Vehicle / 4-Car / 64-Transport / 178-Samsung Galaxy / 899-Samsung Galaxy S III / 4290-Engadget / \n",
"1-Vehicle / 4-Car / 41-Sports car / 349-Chevrolet / 509-Classic car / 650-Convertible / 952-Chevrolet Corvette / \n",
"93-Comedy / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 72-iPhone / \n",
"3-Concert / \n",
"115-Ballet / 350-Circus / 2358-Contortion / \n",
"1-Vehicle / 4-Car / 207-Exhaust system / 245-BMW / \n",
"66-Bollywood / \n",
"0-Games / 2-Video game / 1777-Ratchet / 1824-Clank / 1859-Ratchet & Clank / \n",
"1-Vehicle / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 197-Trail / 220-Mountain bike / 284-Mountain biking / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / 39-Drums / 44-Drums / 296-Saxophone / 1771-The Pink Panther / \n",
"121-Photography / 1046-Remote control / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / \n",
"259-Fire / \n",
"0-Games / 2-Video game / 20-Trailer / 2056-Battleship / \n",
"122-Weight training / 214-Muscle / 295-Chocolate / 841-Butter / 1162-Dietary supplement / 1814-Peanut / 2197-Peanut butter / 2681-Marshmallow / 3498-Whey protein / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / 44-Drums / \n",
"3-Concert / 7-Musician / 14-Guitar / 99-Christmas / \n",
"116-Pokémon / \n",
"66-Bollywood / \n",
"9-Music video / \n",
"445-Guitar Hero / \n",
"45-Cosmetics / 56-Hair / 235-Lipstick / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / 80-Horse / \n",
"6-Animation / 269-Comic book / \n",
"6-Animation / 15-Cartoon / \n",
"3-Concert / 68-Lighting / 4687-Hollywood Palladium / \n",
"12-Food / 391-Chicken meat / 1408-Snack / 2080-Cup / 2943-KFC / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 118-Grand Theft Auto V / 176-PlayStation 4 / \n",
"260-Prayer / 1944-Vatican City / \n",
"5-Dance / 2551-Para Para / \n",
"1-Vehicle / 27-Motorcycle / 105-Motorcycling / \n",
"0-Games / 2-Video game / 19-PC game / 1852-Final Fantasy XI / \n",
"0-Games / 8-Football / 79-American football / \n",
"10-Animal / 132-Bird / 627-Clock / \n",
"695-Balloon / 2099-Balloon modelling / \n",
"9-Music video / \n",
"0-Games / 19-PC game / 51-Strategy video game / \n",
"0-Games / 411-MapleStory / \n",
"1-Vehicle / 4-Car / 625-Factory / \n",
"3-Concert / \n",
"121-Photography / 155-Camera / 566-Digital camera / 629-Camera lens / 1348-Panasonic Corporation / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 40-Road / 64-Transport / 96-Soldier / 105-Motorcycling / \n",
"916-Spa / \n",
"0-Games / 181-Xbox / 386-Xbox / 429-Website / \n",
"6-Animation / 15-Cartoon / 183-Manga / 1312-Dynasty Warriors / 3874-Dynasty Warriors 8 / \n",
"0-Games / 5-Dance / 2985-Chunichi Dragons Doala / \n",
"125-Television / 205-Newscaster / 1572-Husband / \n",
"1288-Marvel vs. Capcom / 1499-Marvel vs. Capcom 3: Fate of Two Worlds / 1505-Ultimate Marvel vs. Capcom 3 / 2172-Deadpool / \n",
"102-Building / 339-Architecture / 425-City / 1214-Skyscraper / \n",
"12-Food / 26-Cooking / 354-Egg / 713-Egg / \n",
"14-Guitar / 100-Electric guitar / \n",
"47-Personal computer / 145-Tablet computer / \n",
"1-Vehicle / 4-Car / 263-Light / 403-Toyota / 1194-Automotive lighting / 2512-Toyota Prius / \n",
"5-Dance / \n",
"232-Cake / 1322-Biscuit / \n",
"308-Ship / 2056-Battleship / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / 99-Christmas / \n",
"1-Vehicle / 4-Car / 113-House / \n",
"260-Prayer / \n",
"10-Animal / 17-Racing / 48-Pet / 71-Dog / 80-Horse / \n",
"0-Games / 530-Cue sports / 568-Pool / 994-Cue stick / 1334-Eight-ball / \n",
"0-Games / \n",
"10-Animal / 132-Bird / 404-Livestock / 424-Chicken / 1136-Poultry / 1428-Rooster / \n",
"0-Games / 2-Video game / 229-Halo / \n",
"2-Video game / 43-Call of Duty / 96-Soldier / \n",
"122-Weight training / 214-Muscle / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 1025-Personal water craft / 1336-Jet Ski / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 109-Cooking show / \n",
"1-Vehicle / 11-Motorsport / 167-Four-wheel drive / 2742-Mini 4WD / \n",
"0-Games / 1-Vehicle / 25-Toy / 62-Train / 119-Locomotive / 507-Thomas the Tank Engine / 1279-Plarail / 1685-Toy train / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"0-Games / 51-Strategy video game / 373-Clash of Clans / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"6-Animation / 116-Pokémon / 1104-Ash Ketchum / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 81-Athlete / 1247-Penalty kick / \n",
"126-The Walt Disney Company / 144-Amusement park / 576-Walt Disney World / \n",
"1-Vehicle / 4-Car / 299-Volkswagen Passenger Cars / 668-Volkswagen Golf / 2147-Fiber / \n",
"9-Music video / \n",
"6-Animation / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"5-Dance / 9-Music video / \n",
"22-Nature / 86-Plant / 137-Tree / 155-Camera / 213-Forest / \n",
"0-Games / 382-Dota 2 / \n",
"28-Fashion / 206-Shoe / 333-Nike; Inc. / 368-Sneakers / 737-Joker / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 40-Road / \n",
"83-Skateboarding / 108-Skateboard / 555-Longboard / 667-Longboarding / 1654-Peter Pan / \n",
"0-Games / 59-Winter sport / 112-Ice skating / 142-Hockey / 312-Ice rink / \n",
"1144-Castlevania / \n",
"75-Wrestling / 85-Combat / \n",
"130-Gymnastics / 1417-Balance beam / \n",
"5-Dance / 49-School / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / \n",
"0-Games / 35-Minecraft / 102-Building / 113-House / 339-Architecture / \n",
"20-Trailer / \n",
"0-Games / 8-Football / 21-Stadium / 77-Arena / \n",
"5-Dance / \n",
"5-Dance / 115-Ballet / \n",
"0-Games / 2-Video game / 6-Animation / 185-Dragon Ball / 244-Goku / 418-Gohan / 2144-Dragon Ball: Raging Blast 2 / 2384-Dragon Ball: Raging Blast / \n",
"2414-Firefighting / \n",
"51-Strategy video game / 271-Tank / 409-World of Tanks / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 20-Trailer / 294-Batman / 630-Superman / 803-DC Comics / 1164-Cartoon Network / 2766-Lex Luthor / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 47-Personal computer / 72-iPhone / 101-Telephone / 145-Tablet computer / 178-Samsung Galaxy / 487-Samsung Electronics / 1015-Samsung Galaxy Note series / 1629-Samsung Galaxy Note / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 158-Tire / 378-Drag racing / \n",
"18-Outdoor recreation / 144-Amusement park / 321-Roller coaster / 334-Amusement ride / \n",
"8-Football / \n",
"0-Games / 8-Football / 21-Stadium / 125-Television / \n",
"9-Music video / \n",
"0-Games / 8-Football / 76-Ball / \n",
"544-Table tennis / 1615-Pong / \n",
"10-Animal / 42-Fishing / 91-Fish / \n",
"7-Musician / 46-Choir / 78-Wedding / \n",
"6-Animation / 15-Cartoon / \n",
"20-Trailer / \n",
"6-Animation / 15-Cartoon / \n",
"185-Dragon Ball / 244-Goku / 869-Dragon Ball Z: Budokai Tenkaichi / \n",
"3-Concert / \n",
"6-Animation / 25-Toy / 477-Transformers / 990-Optimus Prime / \n",
"6-Animation / 15-Cartoon / 3791-Magi: The Labyrinth of Magic / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 138-Call of Duty: Black Ops II / \n",
"20-Trailer / \n",
"2357-Vampire Knight / \n",
"28-Fashion / 469-Bag / 676-Handbag / \n",
"3-Concert / \n",
"3-Concert / 1854-Centella / \n",
"49-School / 56-Hair / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / \n",
"0-Games / 1290-Dragon Nest / 2043-Mabinogi / \n",
"15-Cartoon / 61-Art / 97-Drawing / 382-Dota 2 / 408-Sketch / 643-Hero / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 94-Dashcam / 120-Winter / 725-Human / 915-Town / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 38-Orchestra / 44-Drums / 1240-Mambo / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 25-Toy / 92-Radio-controlled model / 202-Radio-controlled car / \n",
"1-Vehicle / 4-Car / 23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 245-BMW / \n",
"0-Games / 2-Video game / 20-Trailer / 3078-Metro: Last Light / \n",
"3-Concert / \n",
"6-Animation / 15-Cartoon / 123-Naruto / 285-Sasuke Uchiha / \n",
"3-Concert / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / 1044-Coca-Cola / 1224-Cola / \n",
"661-Stretching / 2582-Split / \n",
"3-Concert / 7-Musician / \n",
"115-Ballet / 980-Solo dance / \n",
"6-Animation / 15-Cartoon / 1737-Dance Dance Revolution / 1911-StepMania / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 92-Radio-controlled model / 202-Radio-controlled car / 1040-Street racing / \n",
"0-Games / 739-Guild Wars / \n",
"9-Music video / \n",
"3-Concert / 68-Lighting / \n",
"0-Games / 60-Basketball / \n",
"10-Animal / 48-Pet / \n",
"6-Animation / 123-Naruto / 285-Sasuke Uchiha / \n",
"18-Outdoor recreation / 347-Climbing / 828-Rock climbing / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 132-Bird / 217-Hunting / 280-Wildlife / 2333-Wild turkey / 4351-Turkey / \n",
"356-Star / 3096-Hiru TV / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / \n",
"6-Animation / 10-Animal / 987-Sheep / \n",
"0-Games / 6-Animation / \n",
"208-Wood / 261-Tool / 1226-Metalworking / 2035-Forge / \n",
"0-Games / 2-Video game / 72-iPhone / 189-iPad / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 2249-Driver / \n",
"1237-Baby carriage / \n",
"5-Dance / 49-School / 95-Talent show / \n",
"162-Medicine / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 378-Drag racing / \n",
"5-Dance / 25-Toy / 115-Ballet / 323-Toddler / \n",
"0-Games / \n",
"0-Games / 2-Video game / \n",
"2-Video game / 34-Action-adventure game / 483-Costume / \n",
"248-Hotel / \n",
"434-Ninja / \n",
"3-Concert / 31-Disc jockey / \n",
"25-Toy / 323-Toddler / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / 40-Road / 67-Cycling / 69-Bicycle / 94-Dashcam / 105-Motorcycling / 212-Highway / 608-Wheelie / \n",
"248-Hotel / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 81-Athlete / \n",
"18-Outdoor recreation / 33-Weapon / 217-Hunting / 1873-Crossbow / \n",
"8-Football / 880-Trousers / \n",
"5-Dance / 679-Mickey Mouse / 1234-Mouse / 1327-Minnie Mouse / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 111-PlayStation 3 / 704-Call of Duty: World at War / \n",
"1-Vehicle / 18-Outdoor recreation / 22-Nature / 67-Cycling / 69-Bicycle / 197-Trail / 220-Mountain bike / 284-Mountain biking / \n",
"5-Dance / 16-Performance art / 95-Talent show / 115-Ballet / \n",
"20-Trailer / 325-Vampire / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 79-American football / 993-Quarterback / \n",
"7-Musician / 14-Guitar / \n",
"359-Harry Potter (Literary Series) / \n",
"3-Concert / 7-Musician / 14-Guitar / 30-Drummer / \n",
"5-Dance / 16-Performance art / 95-Talent show / \n",
"10-Animal / 80-Horse / 247-Stallion / 289-Dressage / \n",
"373-Clash of Clans / \n",
"6-Animation / 15-Cartoon / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"6-Animation / 25-Toy / 140-LEGO / 940-Clay animation / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 73-Ball / 76-Ball / 77-Arena / 81-Athlete / \n",
"10-Animal / 1568-Koi / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 96-Soldier / 194-Call of Duty: Modern Warfare 2 / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 60-Basketball / 77-Arena / 168-Basketball moves / 255-Slam dunk / \n",
"3-Concert / \n",
"116-Pokémon / 690-Pokémon X and Y / \n",
"5-Dance / 16-Performance art / 115-Ballet / \n",
"59-Winter sport / 177-Skiing / 789-Alpine skiing / \n",
"28-Fashion / 45-Cosmetics / 195-Mascara / \n",
"61-Art / 242-Paper / 2424-Greeting card / \n",
"0-Games / 2-Video game / 35-Minecraft / 174-Farm / 424-Chicken / \n",
"67-Cycling / 69-Bicycle / 377-BMX bike / \n",
"9-Music video / \n",
"3-Concert / 68-Lighting / \n",
"86-Plant / 137-Tree / \n",
"45-Cosmetics / 238-Nail / 300-Nail art / 304-Nail polish / 338-Manicure / \n",
"159-World of Warcraft / 171-Warcraft / \n",
"5-Dance / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"30-Drummer / 39-Drums / 44-Drums / 3066-Timbales / \n",
"1013-Walking / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 245-BMW / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 96-Soldier / 192-Battlefield / 383-Battlefield 4 / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"0-Games / 211-RuneScape / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 189-iPad / 272-iPod / 331-iPod touch / \n",
"6-Animation / 9-Music video / 781-Pachinko / \n",
"25-Toy / 587-Clay / 2499-Christmas ornament / \n",
"8-Football / 21-Stadium / 1488-Mosaic / \n",
"9-Music video / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / 224-Rapid transit / \n",
"0-Games / 1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 294-Batman / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 50-Aircraft / 82-Airplane / 129-Aviation / 151-Landing / 223-Weather / 451-Cockpit / \n",
"75-Wrestling / 269-Comic book / \n",
"159-World of Warcraft / 171-Warcraft / 1356-Paladin / \n",
"47-Personal computer / 131-Computer / 141-Microsoft Windows / 228-Laptop / 309-Computer hardware / 621-Asus / 1344-Netbook / \n",
"97-Drawing / 567-Compact disc / 1010-Ring / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"56-Hair / 106-Hairstyle / \n",
"98-Festival / 144-Amusement park / 348-Fruit / \n",
"56-Hair / 106-Hairstyle / 609-Tattoo / 863-Body piercing / \n",
"153-Dress / \n",
"93-Comedy / \n",
"9-Music video / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 239-Roasting / 454-Sauce / 671-Tomato / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 391-Chicken meat / 424-Chicken / 597-Soup / \n",
"0-Games / 2-Video game / 43-Call of Duty / 47-Personal computer / 111-PlayStation 3 / 134-Call of Duty: Black Ops / 394-Map / \n",
"3-Concert / 7-Musician / 30-Drummer / 68-Lighting / \n",
"0-Games / 1138-Tower of Saviors / \n",
"5-Dance / 95-Talent show / \n",
"12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / 1218-Japanese Cuisine / 1227-Sushi / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 1559-Spyro / \n",
"6-Animation / 15-Cartoon / \n",
"25-Toy / 89-Comics / 269-Comic book / 1676-Aikatsu! / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 178-Samsung Galaxy / 1366-LG Optimus series / 3962-LG G series / 4407-LG Optimus G / \n",
"0-Games / 35-Minecraft / \n",
"0-Games / 75-Wrestling / 777-WWE 2K / \n",
"66-Bollywood / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 84-Snow / 262-Ice / \n",
"1-Vehicle / 4-Car / 114-Engine / \n",
"86-Plant / 136-Gardening / 184-Vegetable / 279-Garden / 1381-Garlic / \n",
"208-Wood / \n",
"2531-Dojo / \n",
"0-Games / 8-Football / 65-Kick / \n",
"3-Concert / 68-Lighting / \n",
"36-Piano / 53-Keyboard / 4443-Jekyll & Hyde / \n",
"1-Vehicle / 4-Car / 365-Fireworks / \n",
"0-Games / 1-Vehicle / 25-Toy / 62-Train / 140-LEGO / 161-Train station / \n",
"7-Musician / 14-Guitar / 24-String instrument / \n",
"5-Dance / \n",
"0-Games / \n",
"126-The Walt Disney Company / 996-Princess / \n",
"222-Surfing / 815-Surfboard / 1667-Standup paddleboarding / \n",
"12-Food / \n",
"51-Strategy video game / 831-Plants vs. Zombies / 1120-Tower defense / \n",
"1-Vehicle / \n",
"419-Earth / 448-Planet / \n",
"0-Games / 33-Weapon / 257-Counter-Strike / \n",
"10-Animal / 48-Pet / 71-Dog / 85-Combat / 1123-Domestic pig / 1215-Wild boar / \n",
"9-Music video / \n",
"356-Star / 419-Earth / 448-Planet / 654-Sun / 2761-Venus / \n",
"0-Games / 2-Video game / 43-Call of Duty / 138-Call of Duty: Black Ops II / \n",
"0-Games / 2-Video game / 257-Counter-Strike / 315-Knife / \n",
"61-Art / 457-Knitting / 587-Clay / \n",
"88-Machine / \n",
"5-Dance / 16-Performance art / 66-Bollywood / \n",
"1-Vehicle / 4-Car / 683-Renault / \n",
"131-Computer / 141-Microsoft Windows / \n",
"6-Animation / 15-Cartoon / 185-Dragon Ball / 244-Goku / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 127-Xbox 360 / 216-Call of Duty: Modern Warfare 3 / 2491-Spec Ops: The Line / \n",
"1-Vehicle / 4-Car / 49-School / 62-Train / 64-Transport / 286-Bus / 2199-School bus / \n",
"0-Games / 2-Video game / 6-Animation / 34-Action-adventure game / 243-Sonic the Hedgehog / 4019-Sonic the Hedgehog CD / \n",
"6-Animation / 123-Naruto / 622-Sakura Haruno / 4333-Kiba Inuzuka / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / \n",
"9-Music video / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 178-Samsung Galaxy / 899-Samsung Galaxy S III / 1078-Installation art / \n",
"5-Dance / \n",
"3-Concert / \n",
"4646-Tweety / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 135-Drifting / 245-BMW / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"144-Amusement park / 1731-Haunted house / 4274-Knott's Berry Farm / \n",
"12-Food / 26-Cooking / \n",
"3-Concert / 7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / 670-Banjo / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"10-Animal / 48-Pet / 132-Bird / 1799-Domestic canary / \n",
"0-Games / 2-Video game / 5-Dance / 199-Wii / 2123-Just Dance / 3902-Just Dance 3 / \n",
"155-Camera / 629-Camera lens / 4195-Binoculars / \n",
"59-Winter sport / 177-Skiing / \n",
"122-Weight training / 214-Muscle / 1162-Dietary supplement / \n",
"246-Furniture / \n",
"3-Concert / 16-Performance art / \n",
"3-Concert / \n",
"6-Animation / 15-Cartoon / 750-Sailor Moon / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"3-Concert / 13-Musical ensemble / 16-Performance art / \n",
"227-Water / 640-Bottle / \n",
"2-Video game / 6-Animation / 362-PlayStation Portable / 1492-Persona / 2544-Megami Tensei / 2896-Shin Megami Tensei: Persona 3 / \n",
"102-Building / 248-Hotel / 369-Resort / \n",
"1-Vehicle / 4-Car / 336-Scooter / \n",
"1-Vehicle / 4-Car / 1074-Hyundai / \n",
"0-Games / 2-Video game / 43-Call of Duty / 138-Call of Duty: Black Ops II / 3188-Camouflage / \n",
"0-Games / 2-Video game / 43-Call of Duty / \n",
"12-Food / 32-Recipe / 109-Cooking show / 534-Milk / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 570-Road bicycle racing / 645-Road bicycle / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 927-Supermoto / \n",
"93-Comedy / 125-Television / \n",
"28-Fashion / 390-Model / \n",
"0-Games / 2-Video game / 19-PC game / 159-World of Warcraft / 171-Warcraft / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 99-Christmas / 105-Motorcycling / \n",
"7-Musician / 14-Guitar / 30-Drummer / 39-Drums / 44-Drums / 146-Snare drum / \n",
"1-Vehicle / 4-Car / 92-Radio-controlled model / 114-Engine / 154-Wheel / 202-Radio-controlled car / \n",
"102-Building / 353-Wall / \n",
"0-Games / 25-Toy / 162-Medicine / \n",
"10-Animal / 80-Horse / \n",
"9-Music video / \n",
"0-Games / 35-Minecraft / \n",
"1-Vehicle / 124-Tractor / 156-Agriculture / 174-Farm / 715-Farming Simulator / \n",
"6-Animation / 460-Yu-Gi-Oh! Trading Card Game / \n",
"20-Trailer / \n",
"8-Football / 54-Highlight film / 79-American football / \n",
"56-Hair / 61-Art / 97-Drawing / 148-Painting / 162-Medicine / 282-Skin / \n",
"0-Games / 2-Video game / 351-Jumping / \n",
"12-Food / 26-Cooking / 32-Recipe / 236-Baking / 485-Dough / 600-Flour / 662-Pizza / \n",
"10-Animal / 48-Pet / 166-Cat / 346-Kitten / \n",
"78-Wedding / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 133-Nightclub / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"126-The Walt Disney Company / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / 3812-Dobro / 4377-Resonator guitar / \n",
"9-Music video / 1623-Shaolin Kung Fu / \n",
"1-Vehicle / 50-Aircraft / 92-Radio-controlled model / 149-Model aircraft / 157-Jet aircraft / 160-Radio-controlled aircraft / 266-Helicopter / 392-Radio-controlled helicopter / \n",
"1-Vehicle / 4-Car / 70-Driving / 155-Camera / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"0-Games / 2-Video game / 43-Call of Duty / 96-Soldier / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / \n",
"12-Food / 170-Home improvement / 492-Floor / 666-Concrete / 934-Flooring / 1314-Garage / 2093-Coating / \n",
"6-Animation / 25-Toy / 319-Robot / 477-Transformers / 990-Optimus Prime / 2186-Megatron / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / 224-Rapid transit / 1978-New York City Subway / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 73-Ball / 77-Arena / \n",
"230-Brass instrument / 296-Saxophone / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 184-Vegetable / 239-Roasting / 274-Meat / 391-Chicken meat / 2178-Chinese food / \n",
"12-Food / 26-Cooking / 32-Recipe / 179-Kitchen / 315-Knife / 2017-Sharpening / 3880-Knife sharpening / \n",
"10-Animal / 80-Horse / 351-Jumping / \n",
"162-Medicine / 925-Brain / 1925-Skull / \n",
"122-Weight training / 1162-Dietary supplement / \n",
"28-Fashion / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 38-Orchestra / \n",
"0-Games / 1-Vehicle / 25-Toy / \n",
"28-Fashion / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / 49-School / \n",
"0-Games / 8-Football / \n",
"93-Comedy / \n",
"96-Soldier / 205-Newscaster / \n",
"56-Hair / 106-Hairstyle / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 24-String instrument / 30-Drummer / 38-Orchestra / 39-Drums / \n",
"2864-Monogatari Series / \n",
"49-School / \n",
"12-Food / 26-Cooking / 32-Recipe / 1174-Smoothie / \n",
"6-Animation / 2808-Printed circuit board / \n",
"68-Lighting / 99-Christmas / 263-Light / 729-Christmas decoration / 1098-Christmas lights / \n",
"83-Skateboarding / 108-Skateboard / 555-Longboard / 667-Longboarding / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 114-Engine / 349-Chevrolet / 1752-Chevrolet SS / 2061-Chevrolet Chevelle / \n",
"9-Music video / \n",
"1944-Vatican City / \n",
"28-Fashion / 2319-Zara / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 76-Ball / 710-Handball / \n",
"66-Bollywood / \n",
"130-Gymnastics / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / 1458-Electronic drum / 1953-Roland V-Drums / \n",
"23-Mobile phone / 29-Smartphone / 47-Personal computer / 145-Tablet computer / 178-Samsung Galaxy / 487-Samsung Electronics / 1287-Samsung Galaxy Tab series / \n",
"1-Vehicle / 213-Forest / 259-Fire / 371-Firefighter / 796-Fire engine / \n",
"0-Games / 2-Video game / 8-Football / 90-Sports game / 180-FIFA 15 / 3235-Monopoly / \n",
"273-News program / \n",
"5-Dance / 49-School / \n",
"22-Nature / \n",
"6-Animation / 15-Cartoon / 126-The Walt Disney Company / 708-Mermaid / 1219-Ariel / \n",
"1-Vehicle / 4-Car / 790-Nissan GT-R / \n",
"6-Animation / 995-Attack on Titan / \n",
"61-Art / 97-Drawing / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"237-Final Fantasy / \n",
"0-Games / 2-Video game / 264-Star Wars / \n",
"1-Vehicle / 18-Outdoor recreation / 59-Winter sport / 84-Snow / 120-Winter / 177-Skiing / 281-Snowboarding / \n",
"9-Music video / 20-Trailer / \n",
"0-Games / 60-Basketball / 81-Athlete / \n",
"23-Mobile phone / 29-Smartphone / 56-Hair / 72-iPhone / 272-iPod / 331-iPod touch / \n",
"227-Water / 1081-Lemon / 1999-Eggplant / \n",
"28-Fashion / \n",
"12-Food / 2706-Starbucks / \n",
"8-Football / \n",
"9-Music video / \n",
"242-Paper / 370-Origami / 1023-Rose / \n",
"6-Animation / \n",
"3-Concert / 16-Performance art / \n",
"10-Animal / 2534-Scorpion / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 34-Action-adventure game / 135-Drifting / 201-Grand Theft Auto: San Andreas / 297-Grand Theft Auto IV / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"66-Bollywood / \n",
"1182-Softball / \n",
"12-Food / 720-Wine / \n",
"56-Hair / 97-Drawing / 106-Hairstyle / 1827-Hair straightening / \n",
"8-Football / 21-Stadium / 595-Champion / \n",
"0-Games / 2-Video game / 51-Strategy video game / 165-League of Legends / \n",
"6-Animation / 15-Cartoon / 116-Pokémon / 1104-Ash Ketchum / 3347-Misty / \n",
"0-Games / 326-Metin2 / 1552-Video lesson / \n",
"0-Games / 8-Football / \n",
"28-Fashion / 56-Hair / \n",
"8-Football / \n",
"0-Games / 25-Toy / 140-LEGO / 852-Lego Star Wars / 884-Lego minifigure / 1478-Lego Ninjago / \n",
"0-Games / 2-Video game / 1492-Persona / 2139-Dungeon crawl / \n",
"1-Vehicle / 4-Car / 3219-Daihatsu / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / \n",
"3-Concert / 68-Lighting / 263-Light / \n",
"46-Choir / \n",
"6-Animation / 31-Disc jockey / \n",
"242-Paper / 370-Origami / 631-Cube / \n",
"8-Football / 21-Stadium / \n",
"28-Fashion / \n",
"0-Games / 60-Basketball / 168-Basketball moves / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 932-Microsoft Lumia / \n",
"20-Trailer / \n",
"1-Vehicle / 50-Aircraft / 451-Cockpit / 856-Airbus A320 family / \n",
"3-Concert / 16-Performance art / 133-Nightclub / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"3-Concert / 7-Musician / \n",
"6-Animation / \n",
"1-Vehicle / 4-Car / 1661-Volkswagen Gol / \n",
"0-Games / 599-Dofus / \n",
"56-Hair / 106-Hairstyle / 3584-Hair roller / \n",
"7-Musician / 13-Musical ensemble / 21-Stadium / 79-American football / 203-Marching band / 209-University / \n",
"0-Games / 2-Video game / 6-Animation / 816-Gundam / 1564-Mobile Suit Gundam: Extreme Vs. / \n",
"5-Dance / \n",
"67-Cycling / 69-Bicycle / 377-BMX bike / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / 1281-Couch / \n",
"28-Fashion / \n",
"1-Vehicle / 62-Train / 64-Transport / 161-Train station / 224-Rapid transit / \n",
"28-Fashion / 45-Cosmetics / \n",
"322-Mixtape / \n",
"1-Vehicle / 50-Aircraft / 139-Airport / 151-Landing / 157-Jet aircraft / 254-Runway / \n",
"9-Music video / 67-Cycling / 69-Bicycle / \n",
"60-Basketball / \n",
"0-Games / 2-Video game / 33-Weapon / \n",
"0-Games / 2-Video game / 33-Weapon / 257-Counter-Strike / \n",
"0-Games / 2-Video game / 756-Tibia / \n",
"1-Vehicle / 11-Motorsport / 87-Boat / 327-Mud / \n",
"3-Concert / 1562-Viola caipira / 2649-Nativity scene / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / \n",
"6-Animation / 10-Animal / 15-Cartoon / \n",
"93-Comedy / \n",
"1-Vehicle / 4-Car / 517-Audi / \n",
"0-Games / 19-PC game / 837-Roblox / \n",
"1-Vehicle / 11-Motorsport / 124-Tractor / 156-Agriculture / 252-Tractor pulling / \n",
"3-Concert / 7-Musician / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 105-Motorcycling / 172-Motocross / \n",
"47-Personal computer / 228-Laptop / 621-Asus / \n",
"6-Animation / 15-Cartoon / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"61-Art / 97-Drawing / 479-Graffiti / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 184-Vegetable / 210-Cookware and bakeware / 239-Roasting / 274-Meat / 688-Beef / 1509-Stew / 3008-Slow cooker / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 1520-Minibike / \n",
"5-Dance / \n",
"6-Animation / 15-Cartoon / 183-Manga / 4231-Slayers / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 431-Call of Duty: Advanced Warfare / 2752-God Mode / \n",
"31-Disc jockey / \n",
"1604-Satan / \n",
"0-Games / 10-Animal / 80-Horse / 310-The Sims / 328-Simulation video game / 564-The Sims 3 / 4083-The Sims 3: Pets / \n",
"5-Dance / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / \n",
"1-Vehicle / 74-Truck / 88-Machine / 249-Heavy equipment / 433-Construction / \n",
"0-Games / 34-Action-adventure game / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 89-Comics / 185-Dragon Ball / \n",
"5-Dance / \n",
"18-Outdoor recreation / 137-Tree / 217-Hunting / 315-Knife / \n",
"8-Football / \n",
"0-Games / 23-Mobile phone / 72-iPhone / \n",
"12-Food / 442-Bread / 1943-Bakery / \n",
"43-Call of Duty / 704-Call of Duty: World at War / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / 545-Quartet (ensemble) / \n",
"0-Games / 2-Video game / 8-Football / 90-Sports game / 480-FIFA 13 / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 241-Fighting game / 486-M.U.G.E.N / \n",
"0-Games / 6-Animation / 460-Yu-Gi-Oh! Trading Card Game / 494-Card game / \n",
"12-Food / 727-Curry / \n",
"6-Animation / 15-Cartoon / \n",
"83-Skateboarding / 108-Skateboard / \n",
"1-Vehicle / 4-Car / 167-Four-wheel drive / 198-Off-road vehicle / 277-Sport utility vehicle / 360-Jeep / \n",
"2152-Selfie / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 297-Grand Theft Auto IV / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 94-Dashcam / 212-Highway / 575-Traffic / \n",
"2-Video game / 584-Super Mario Bros. / 831-Plants vs. Zombies / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / 606-String / 738-Guitar amplifier / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 77-Arena / 4270-Old Trafford / \n",
"204-Gym / \n",
"3-Concert / 5-Dance / 16-Performance art / 38-Orchestra / \n",
"8-Football / 767-Goalkeeper / \n",
"1-Vehicle / 4-Car / \n",
"5-Dance / \n",
"61-Art / 97-Drawing / 609-Tattoo / 1834-Mehndi / \n",
"5-Dance / 8-Football / \n",
"66-Bollywood / 78-Wedding / 2429-India TV / \n",
"282-Skin / 3579-Mona Lisa / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 40-Road / 105-Motorcycling / \n",
"3-Concert / 93-Comedy / 355-Comedian / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"5-Dance / 99-Christmas / 729-Christmas decoration / 964-Christmas tree / \n",
"3-Concert / 7-Musician / 24-String instrument / \n",
"1-Vehicle / 4-Car / 332-Mercedes-Benz / 3826-Mercedes-Benz A-Class / 4364-Mercedes-Benz CLA-Class / \n",
"3-Concert / \n",
"2-Video game / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 245-BMW / \n",
"9-Music video / \n",
"0-Games / 412-Monster / 3234-Kixeye; Inc. / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"10-Animal / 86-Plant / 132-Bird / 571-Bear / 1639-Eagle / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / \n",
"7-Musician / 12-Food / 14-Guitar / 389-Restaurant / 720-Wine / 3829-Champagne / \n",
"1-Vehicle / 4-Car / 20-Trailer / \n",
"25-Toy / 264-Star Wars / \n",
"3-Concert / 7-Musician / 551-British Broadcasting Corporation / \n",
"12-Food / 26-Cooking / 1256-Smoking / \n",
"113-House / 225-Lake / \n",
"0-Games / 2-Video game / 19-PC game / 34-Action-adventure game / 43-Call of Duty / 194-Call of Duty: Modern Warfare 2 / 297-Grand Theft Auto IV / 1923-Niko Bellic / \n",
"14-Guitar / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / 198-Off-road vehicle / \n",
"444-Family / \n",
"0-Games / 19-PC game / \n",
"18-Outdoor recreation / 22-Nature / 67-Cycling / 69-Bicycle / 137-Tree / 197-Trail / 213-Forest / 220-Mountain bike / 284-Mountain biking / 2092-Coal / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 189-iPad / 508-iPhone 5 / \n",
"0-Games / 3643-Starbound / \n",
"113-House / 170-Home improvement / 208-Wood / 276-Room / 488-Door / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 787-Monster truck / \n",
"0-Games / 2-Video game / 15-Cartoon / 3747-Wario / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 235-Lipstick / 291-Rouge / 436-Concealer / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 203-Marching band / 820-Bagpipes / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 169-Rallying / 581-PlayStation / 3500-Colin McRae: Dirt / \n",
"1-Vehicle / 766-President of the United States / \n",
"1-Vehicle / 59-Winter sport / 1829-Sledding / 4641-Toboggan / \n",
"122-Weight training / 214-Muscle / \n",
"8-Football / \n",
"8-Football / \n",
"468-Biology / 1544-Laboratory / \n",
"0-Games / 60-Basketball / 255-Slam dunk / \n",
"13-Musical ensemble / \n",
"44-Drums / 98-Festival / 657-Autumn / \n",
"10-Animal / 208-Wood / 512-Printing / 2840-Engraving / 2967-Printmaking / \n",
"130-Gymnastics / 163-Running / 1335-Trampolining / \n",
"93-Comedy / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 148-Painting / 272-iPod / 331-iPod touch / \n",
"6-Animation / 15-Cartoon / \n",
"1-Vehicle / 4-Car / 64-Transport / 74-Truck / 158-Tire / 249-Heavy equipment / 822-Crane / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 24-String instrument / 38-Orchestra / \n",
"3-Concert / 16-Performance art / 3988-Talk box / \n",
"3129-Sauna / \n",
"79-American football / \n",
"2006-BlazBlue / 2936-BlazBlue: Calamity Trigger / \n",
"0-Games / 2-Video game / 19-PC game / 739-Guild Wars / 839-Guild Wars 2 / \n",
"5-Dance / 1549-Mime artist / \n",
"6-Animation / 15-Cartoon / \n",
"1-Vehicle / 4-Car / 167-Four-wheel drive / 1540-Station Wagon / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 118-Grand Theft Auto V / \n",
"0-Games / 2-Video game / 19-PC game / 1196-Dragon Age / 1735-Dragon Age: Inquisition / 2700-Dragon Age II / \n",
"1-Vehicle / 12-Food / 88-Machine / 666-Concrete / \n",
"10-Animal / 22-Nature / 137-Tree / 280-Wildlife / 846-Squirrel / 3104-Tree squirrel / \n",
"0-Games / 2-Video game / 55-Video game console / 116-Pokémon / 292-Handheld game console / 1153-Pokémon HeartGold and SoulSilver / \n",
"5-Dance / 133-Nightclub / 613-Bar / 1065-Bartender / \n",
"0-Games / 2-Video game / \n",
"1669-Vodka / 3505-Wellington boot / \n",
"12-Food / 26-Cooking / 32-Recipe / 236-Baking / 442-Bread / \n",
"110-Album / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 118-Grand Theft Auto V / 1790-Gold medal / \n",
"810-Village / \n",
"75-Wrestling / \n",
"9-Music video / 237-Final Fantasy / 2691-Tifa Lockhart / \n",
"3-Concert / 7-Musician / 46-Choir / \n",
"5-Dance / 1110-Cell / \n",
"78-Wedding / \n",
"12-Food / 720-Wine / \n",
"222-Surfing / 2415-Big wave surfing / \n",
"10-Animal / 48-Pet / 71-Dog / 166-Cat / \n",
"0-Games / 2-Video game / 413-PlayStation 2 / 1492-Persona / \n",
"3579-Mona Lisa / \n",
"5-Dance / 7-Musician / 13-Musical ensemble / 38-Orchestra / 115-Ballet / 200-Doll / \n",
"6-Animation / 15-Cartoon / 979-Star Trek / \n",
"20-Trailer / 4603-Frankenstein / \n",
"130-Gymnastics / \n",
"0-Games / 792-DarkOrbit / \n",
"12-Food / 28-Fashion / 56-Hair / 106-Hairstyle / \n",
"0-Games / 1-Vehicle / 124-Tractor / 174-Farm / 252-Tractor pulling / 715-Farming Simulator / \n",
"0-Games / 1816-Fire Emblem / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 135-Drifting / \n",
"5-Dance / 18-Outdoor recreation / 287-Human swimming / 329-Swimming pool / 1711-Synchronized swimming / \n",
"6-Animation / 15-Cartoon / 183-Manga / 750-Sailor Moon / 1342-Sailor Moon / \n",
"0-Games / 756-Tibia / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 449-Honda / 482-Asphalt / 552-Honda Civic / 3697-Honda K engine / \n",
"0-Games / 59-Winter sport / 112-Ice skating / 142-Hockey / 262-Ice / 312-Ice rink / 1973-Hockey puck / \n",
"0-Games / 8-Football / 73-Ball / 90-Sports game / 180-FIFA 15 / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / 224-Rapid transit / \n",
"66-Bollywood / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 138-Call of Duty: Black Ops II / 431-Call of Duty: Advanced Warfare / \n",
"0-Games / 2-Video game / 192-Battlefield / 374-Battlefield 3 / \n",
"47-Personal computer / 145-Tablet computer / 357-DVD / \n",
"12-Food / 88-Machine / 512-Printing / 625-Factory / \n",
"0-Games / 19-PC game / 33-Weapon / 43-Call of Duty / \n",
"1-Vehicle / 11-Motorsport / 124-Tractor / 156-Agriculture / 249-Heavy equipment / 252-Tractor pulling / 363-Soil / 459-Plough / \n",
"2038-Credit card / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"3-Concert / \n",
"5-Dance / 79-American football / 191-Cheerleading / 203-Marching band / \n",
"1-Vehicle / 27-Motorcycle / \n",
"1-Vehicle / 4-Car / 68-Lighting / 2580-Capacitor / \n",
"648-Clown / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / 2402-Grand Theft Auto / \n",
"10-Animal / 217-Hunting / 280-Wildlife / 636-Rabbit / 3022-Hare / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / \n",
"527-Battery / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 382-Dota 2 / 549-Defense of the Ancients / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 30-Drummer / \n",
"12-Food / 59-Winter sport / 112-Ice skating / 142-Hockey / 4454-Hockey stick / \n",
"9-Music video / \n",
"978-Love song / \n",
"6-Animation / 88-Machine / 319-Robot / 619-Manufacturing / \n",
"10-Animal / 144-Amusement park / 537-Zoo / 1135-Dolphin / \n",
"1-Vehicle / 4-Car / 40-Road / 64-Transport / 94-Dashcam / 212-Highway / \n",
"205-Newscaster / \n",
"7-Musician / 13-Musical ensemble / 230-Brass instrument / \n",
"12-Food / 539-Home appliance / 1439-Refrigerator / \n",
"18-Outdoor recreation / \n",
"1-Vehicle / 4-Car / 84-Snow / 120-Winter / \n",
"3-Concert / 13-Musical ensemble / 253-Carnival / \n",
"9-Music video / 61-Art / \n",
"3-Concert / 7-Musician / 219-Accordion / 1994-Garmon / \n",
"22-Nature / \n",
"363-Soil / \n",
"1-Vehicle / 27-Motorcycle / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 172-Motocross / \n",
"6-Animation / 102-Building / 113-House / 170-Home improvement / 339-Architecture / 416-Apartment / \n",
"12-Food / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 111-PlayStation 3 / 704-Call of Duty: World at War / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"20-Trailer / \n",
"4349-Neptune / \n",
"42-Fishing / \n",
"28-Fashion / 45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 235-Lipstick / \n",
"1-Vehicle / 4-Car / 72-iPhone / 212-Highway / 675-iPhone 4S / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"432-Kingdom Hearts / 3407-Kingdom Hearts III / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"83-Skateboarding / 108-Skateboard / 555-Longboard / 667-Longboarding / \n",
"5-Dance / 49-School / 95-Talent show / \n",
"3-Concert / 8-Football / \n",
"1-Vehicle / 4-Car / 124-Tractor / 234-Ford / \n",
"88-Machine / 319-Robot / 521-Metal / 619-Manufacturing / 625-Factory / 938-Steel / 2745-Beam / \n",
"28-Fashion / 45-Cosmetics / \n",
"3-Concert / 144-Amusement park / \n",
"0-Games / 2-Video game / 19-PC game / 734-DayZ / 1216-DayZ / \n",
"12-Food / 26-Cooking / 32-Recipe / 274-Meat / 862-Pork / 1256-Smoking / 1557-Bacon / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 189-iPad / 272-iPod / 331-iPod touch / \n",
"0-Games / 159-World of Warcraft / 171-Warcraft / \n",
"121-Photography / \n",
"9-Music video / \n",
"230-Brass instrument / 296-Saxophone / \n",
"209-University / 913-President / \n",
"593-Moon / 654-Sun / \n",
"0-Games / 2-Video game / 35-Minecraft / 394-Map / \n",
"5-Dance / \n",
"5-Dance / 16-Performance art / \n",
"355-Comedian / \n",
"28-Fashion / 206-Shoe / \n",
"9-Music video / 1866-Katara / 3862-Toph Beifong / \n",
"5-Dance / 10-Animal / 80-Horse / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 545-Quartet (ensemble) / \n",
"278-GoPro / 1358-Canyon / \n",
"20-Trailer / \n",
"0-Games / 19-PC game / 51-Strategy video game / 756-Tibia / \n",
"9-Music video / 310-The Sims / 543-The Sims 2 / \n",
"3-Concert / 7-Musician / 14-Guitar / 30-Drummer / 39-Drums / 44-Drums / \n",
"78-Wedding / 190-Bride / 293-Slide show / 421-Gown / 2399-Collage / \n",
"5-Dance / 16-Performance art / 1039-Tiger / \n",
"0-Games / \n",
"401-Cello / \n",
"20-Trailer / \n",
"0-Games / 2-Video game / 264-Star Wars / 2117-Star Wars Battlefront / 2340-Star Wars: Battlefront II / 2891-Star Wars: Battlefront / \n",
"1-Vehicle / 50-Aircraft / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 393-Unmanned aerial vehicle / 604-Quadcopter / \n",
"0-Games / 8-Football / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 391-Chicken meat / \n",
"28-Fashion / 1038-Hijab / 1096-Scarf / 2448-Shawl / \n",
"60-Basketball / 255-Slam dunk / \n",
"0-Games / 2-Video game / 19-PC game / 464-First-person Shooter / \n",
"275-Diving / 381-Underwater diving / 407-Scuba diving / \n",
"35-Minecraft / \n",
"6-Animation / 185-Dragon Ball / 244-Goku / 876-Freeza / \n",
"0-Games / 2-Video game / 19-PC game / 330-Money / 1027-Terraria / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 151-Landing / 175-Airliner / 188-Airline / 254-Runway / 451-Cockpit / \n",
"8-Football / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / \n",
"2-Video game / 55-Video game console / 314-Xbox One / \n",
"258-Aquarium / 288-Underwater / 601-Shark / \n",
"10-Animal / 91-Fish / 258-Aquarium / 1368-Gallon / \n",
"0-Games / 664-CrossFire / \n",
"0-Games / 2-Video game / 90-Sports game / 561-Madden NFL / \n",
"567-Compact disc / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 134-Call of Duty: Black Ops / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 2999-Sleeping Dogs / \n",
"13-Musical ensemble / 38-Orchestra / \n",
"59-Winter sport / 112-Ice skating / 115-Ballet / 262-Ice / 316-Figure skating / 632-Ice dancing / 833-Ice skate / \n",
"7-Musician / 9-Music video / \n",
"0-Games / 2-Video game / 51-Strategy video game / 165-League of Legends / \n",
"10-Animal / 48-Pet / 132-Bird / \n",
"3-Concert / 14-Guitar / 24-String instrument / 150-Violin / \n",
"707-Roof / 794-Bridge / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 651-Stock car racing / 728-Dirt track racing / \n",
"5-Dance / 98-Festival / \n",
"8-Football / 21-Stadium / \n",
"9-Music video / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 57-Race track / 1400-rFactor / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 57-Race track / 135-Drifting / 482-Asphalt / 3636-Toyota JZ engine / \n",
"7-Musician / 14-Guitar / \n",
"1-Vehicle / 4-Car / 234-Ford / \n",
"458-Gold / \n",
"0-Games / 2-Video game / 6-Animation / 19-PC game / 460-Yu-Gi-Oh! Trading Card Game / 4280-Jaden Yuki / \n",
"14-Guitar / \n",
"61-Art / 155-Camera / 479-Graffiti / 1385-Street art / 1846-Mural / 2898-Canon EF lens mount / \n",
"6-Animation / 15-Cartoon / 547-Bleach / \n",
"5-Dance / 95-Talent show / \n",
"264-Star Wars / 717-Darth Vader / 4117-Leia Organa / \n",
"0-Games / 2-Video game / 1566-Doom / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 116-Pokémon / 186-Pokémon / \n",
"46-Choir / \n",
"18-Outdoor recreation / 147-River / \n",
"8-Football / 333-Nike; Inc. / 757-Boot / \n",
"0-Games / 19-PC game / 35-Minecraft / \n",
"76-Ball / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 38-Orchestra / \n",
"0-Games / 2-Video game / 6-Animation / 2431-Metal Slug / 2597-Sprite / 3217-Neo Geo / \n",
"5-Dance / 204-Gym / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 664-CrossFire / \n",
"1-Vehicle / 11-Motorsport / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 161-Train station / \n",
"0-Games / 8-Football / \n",
"6-Animation / 15-Cartoon / 366-Dragon / \n",
"36-Piano / 53-Keyboard / 446-Sheet music / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"3-Concert / \n",
"94-Dashcam / \n",
"5-Dance / 95-Talent show / \n",
"31-Disc jockey / 2371-Hot dog / \n",
"156-Agriculture / \n",
"12-Food / 295-Chocolate / 367-Drink / \n",
"31-Disc jockey / \n",
"6-Animation / 9-Music video / 995-Attack on Titan / \n",
"8-Football / \n",
"102-Building / 113-House / 170-Home improvement / 2055-Mansion / \n",
"6-Animation / 15-Cartoon / \n",
"1-Vehicle / 4-Car / 245-BMW / \n",
"3-Concert / 5-Dance / 16-Performance art / 95-Talent show / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / \n",
"5-Dance / 16-Performance art / 115-Ballet / \n",
"2-Video game / 2628-Sly Cooper / \n",
"5-Dance / 130-Gymnastics / 350-Circus / 2940-Trapeze / \n",
"0-Games / 256-Tennis / \n",
"76-Ball / 350-Circus / 788-Juggling / \n",
"0-Games / 51-Strategy video game / 373-Clash of Clans / \n",
"0-Games / 2-Video game / 51-Strategy video game / 165-League of Legends / \n",
"102-Building / 113-House / 339-Architecture / 416-Apartment / 939-Condominium / \n",
"3-Concert / 68-Lighting / \n",
"10-Animal / 80-Horse / 247-Stallion / 289-Dressage / 438-Mare / \n",
"5-Dance / 16-Performance art / 95-Talent show / \n",
"5-Dance / 115-Ballet / \n",
"31-Disc jockey / 133-Nightclub / \n",
"5-Dance / 16-Performance art / 95-Talent show / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"516-Headphones / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 96-Soldier / 948-Far Cry / 1683-Far Cry 4 / \n",
"31-Disc jockey / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 24-String instrument / 38-Orchestra / 150-Violin / 401-Cello / 450-Viola / \n",
"10-Animal / 48-Pet / 132-Bird / 598-Parrot / 1341-Parakeet / \n",
"191-Cheerleading / \n",
"121-Photography / 155-Camera / 566-Digital camera / 629-Camera lens / 721-Digital SLR / 3328-Canon EOS 7D / \n",
"162-Medicine / 1110-Cell / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 1839-Rottweiler / \n",
"3-Concert / \n",
"5-Dance / 78-Wedding / 153-Dress / 190-Bride / 421-Gown / 590-Wedding dress / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 239-Roasting / 671-Tomato / \n",
"9-Music video / \n",
"3-Concert / \n",
"5-Dance / 1544-Laboratory / \n",
"3-Concert / \n",
"3-Concert / \n",
"0-Games / 2-Video game / \n",
"6-Animation / 10-Animal / 20-Trailer / 91-Fish / \n",
"5-Dance / \n",
"12-Food / 28-Fashion / 56-Hair / 106-Hairstyle / \n",
"66-Bollywood / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"2065-Tsubasa: Reservoir Chronicle / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"1-Vehicle / 4-Car / 154-Wheel / 234-Ford / 918-Brake / 2899-Ford Explorer / \n",
"22-Nature / 23-Mobile phone / 29-Smartphone / 223-Weather / 1379-HTC One / 2328-HTC One (M8) / \n",
"3-Concert / 38-Orchestra / \n",
"0-Games / 59-Winter sport / 81-Athlete / 112-Ice skating / 142-Hockey / 312-Ice rink / \n",
"0-Games / 2-Video game / 55-Video game console / 320-Super Smash Bros. / 443-Wii U / 697-Super Smash Bros. for Nintendo 3DS and Wii U / \n",
"12-Food / 179-Kitchen / 2153-Sink / 2243-Stainless steel / \n",
"12-Food / 227-Water / \n",
"2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 1379-HTC One / \n",
"6-Animation / 15-Cartoon / 61-Art / 89-Comics / 123-Naruto / 183-Manga / 285-Sasuke Uchiha / \n",
"25-Toy / 140-LEGO / \n",
"3-Concert / 7-Musician / \n",
"15-Cartoon / 126-The Walt Disney Company / 2058-Snowman / 2145-Olaf / \n",
"3-Concert / 68-Lighting / \n",
"1-Vehicle / 42-Fishing / 82-Airplane / 778-Gliding / 1263-Glider / \n",
"0-Games / 2-Video game / 6-Animation / 362-PlayStation Portable / 3813-Dynasty Warriors: Gundam / \n",
"3-Concert / 7-Musician / \n",
"6-Animation / 616-The Bible / \n",
"2-Video game / 55-Video game console / 127-Xbox 360 / 181-Xbox / \n",
"313-Book / \n",
"6-Animation / 89-Comics / 183-Manga / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / 670-Banjo / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"12-Food / 215-Eating / 323-Toddler / \n",
"916-Spa / \n",
"5-Dance / 95-Talent show / 98-Festival / \n",
"115-Ballet / \n",
"0-Games / 2-Video game / 685-Luigi / 887-King Koopa / 4464-Mario & Luigi: Bowser's Inside Story / \n",
"12-Food / 32-Recipe / 226-Dessert / 232-Cake / 236-Baking / 295-Chocolate / 1189-Torte / \n",
"6-Animation / 15-Cartoon / 1118-Kirby / 2266-Potato chip / \n",
"5-Dance / 16-Performance art / 95-Talent show / \n",
"0-Games / 39-Drums / 356-Star / 445-Guitar Hero / 724-Rock Band / \n",
"1-Vehicle / 27-Motorcycle / 40-Road / 94-Dashcam / 158-Tire / \n",
"55-Video game console / 111-PlayStation 3 / 414-Computer monitor / \n",
"12-Food / 238-Nail / \n",
"3-Concert / 7-Musician / 16-Performance art / 30-Drummer / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 133-Nightclub / \n",
"0-Games / 8-Football / 21-Stadium / 77-Arena / \n",
"28-Fashion / 164-Beach / \n",
"12-Food / 26-Cooking / 32-Recipe / 109-Cooking show / 179-Kitchen / 232-Cake / 1189-Torte / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / \n",
"1-Vehicle / 87-Boat / 308-Ship / 917-Sail / 1309-Sailing ship / \n",
"122-Weight training / 204-Gym / \n",
"0-Games / 2-Video game / 19-PC game / 34-Action-adventure game / 35-Minecraft / \n",
"209-University / 2015-News broadcasting / \n",
"9-Music video / \n",
"47-Personal computer / 131-Computer / 141-Microsoft Windows / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 19-PC game / 739-Guild Wars / 839-Guild Wars 2 / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 81-Athlete / \n",
"86-Plant / 136-Gardening / 137-Tree / 1023-Rose / \n",
"9-Music video / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 101-Telephone / 3089-Nokia N8 / \n",
"10-Animal / 80-Horse / 351-Jumping / \n",
"5-Dance / 16-Performance art / \n",
"419-Earth / \n",
"25-Toy / 140-LEGO / 749-Figurine / 855-Hulk / 884-Lego minifigure / 2276-Lego Marvel Super Heroes / \n",
"6-Animation / 89-Comics / 123-Naruto / 444-Family / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 96-Soldier / 422-Call of Duty: Ghosts / \n",
"5-Dance / 16-Performance art / 361-Flamenco / \n",
"9-Music video / 110-Album / \n",
"46-Choir / \n",
"22-Nature / 42-Fishing / 267-Mountain / 573-Paragliding / 800-Mountain pass / \n",
"1-Vehicle / 50-Aircraft / 92-Radio-controlled model / 137-Tree / 149-Model aircraft / 160-Radio-controlled aircraft / 266-Helicopter / 392-Radio-controlled helicopter / \n",
"10-Animal / 280-Wildlife / \n",
"0-Games / 1029-Aion: The Tower of Eternity / \n",
"3-Concert / \n",
"9-Music video / 78-Wedding / 190-Bride / 590-Wedding dress / \n",
"61-Art / 148-Painting / 329-Swimming pool / \n",
"1-Vehicle / 4-Car / 74-Truck / 154-Wheel / 167-Four-wheel drive / 234-Ford / 375-Pickup truck / \n",
"9-Music video / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"3-Concert / 5-Dance / \n",
"9-Music video / \n",
"23-Mobile phone / 2038-Credit card / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 40-Road / 212-Highway / \n",
"0-Games / 8-Football / \n",
"61-Art / 148-Painting / \n",
"12-Food / 26-Cooking / 32-Recipe / 348-Fruit / 985-Strawberry / \n",
"28-Fashion / 59-Winter sport / 281-Snowboarding / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 1019-Christ / \n",
"1-Vehicle / 17-Racing / 67-Cycling / 69-Bicycle / 570-Road bicycle racing / \n",
"31-Disc jockey / \n",
"3-Concert / \n",
"9-Music video / \n",
"1-Vehicle / 25-Toy / 62-Train / 507-Thomas the Tank Engine / 1154-Anpanman / 1685-Toy train / \n",
"1-Vehicle / 4-Car / 25-Toy / 198-Off-road vehicle / 360-Jeep / \n",
"656-Metal Gear / 2698-Metal Gear Rising: Revengeance / \n",
"12-Food / 205-Newscaster / \n",
"10-Animal / 174-Farm / 1116-Goat / \n",
"18-Outdoor recreation / 222-Surfing / 233-Ocean / 4412-Alcatraz Island / \n",
"3-Concert / 68-Lighting / 133-Nightclub / \n",
"12-Food / 367-Drink / 1325-Gel / 2731-Aloe vera / \n",
"200-Doll / 587-Clay / \n",
"0-Games / 2-Video game / 90-Sports game / 127-Xbox 360 / \n",
"10-Animal / 80-Horse / 1323-Helmet camera / \n",
"1-Vehicle / 527-Battery / 2755-Lithium polymer battery / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 30-Drummer / \n",
"22-Nature / 42-Fishing / 147-River / 283-Rock / 825-Stream / \n",
"17-Racing / 67-Cycling / 69-Bicycle / 570-Road bicycle racing / \n",
"31-Disc jockey / \n",
"93-Comedy / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / \n",
"12-Food / 1545-Floristry / \n",
"22-Nature / 260-Prayer / \n",
"209-University / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / 1197-Hedgehog / 1299-Shadow the Hedgehog / 1512-Sonic Generations / 2028-Sonic World Adventure / 3321-Shadow the Hedgehog / \n",
"6-Animation / 15-Cartoon / \n",
"3-Concert / 5-Dance / 2032-Pepsi / \n",
"1-Vehicle / 4-Car / \n",
"162-Medicine / \n",
"6-Animation / 15-Cartoon / 183-Manga / \n",
"191-Cheerleading / \n",
"12-Food / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / 1942-Oboe / \n",
"1-Vehicle / 8-Football / 27-Motorcycle / \n",
"5-Dance / \n",
"20-Trailer / 66-Bollywood / \n",
"457-Knitting / \n",
"9-Music video / 1950-The CW / 3068-Green Arrow / \n",
"5-Dance / 16-Performance art / 95-Talent show / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 81-Athlete / \n",
"587-Clay / \n",
"0-Games / 2-Video game / 116-Pokémon / 186-Pokémon / \n",
"61-Art / 97-Drawing / 148-Painting / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / \n",
"1-Vehicle / 64-Transport / 74-Truck / 158-Tire / \n",
"1-Vehicle / 4-Car / \n",
"66-Bollywood / \n",
"1-Vehicle / 4-Car / 259-Fire / 371-Firefighter / \n",
"290-Pony / 315-Knife / \n",
"3-Concert / 5-Dance / \n",
"2128-Pollution / \n",
"6-Animation / 25-Toy / \n",
"59-Winter sport / 281-Snowboarding / 1008-Kite / 4321-Mountainboarding / \n",
"326-Metin2 / \n",
"3-Concert / 5-Dance / 16-Performance art / 38-Orchestra / \n",
"1-Vehicle / 85-Combat / 271-Tank / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 40-Road / 74-Truck / 137-Tree / 197-Trail / 363-Soil / \n",
"67-Cycling / 69-Bicycle / 669-Extreme sport / 769-Red Bull / \n",
"47-Personal computer / 131-Computer / 309-Computer hardware / 602-Central processing unit / \n",
"0-Games / 411-MapleStory / \n",
"0-Games / 960-Badminton / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 19-PC game / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / 413-PlayStation 2 / 858-Carl Johnson / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 24-String instrument / 150-Violin / 401-Cello / 450-Viola / \n",
"1-Vehicle / 87-Boat / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 169-Rallying / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 41-Sports car / \n",
"0-Games / 35-Minecraft / 488-Door / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 569-Lamborghini / 1132-Lamborghini Aventador / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 89-Comics / 123-Naruto / 591-Naruto: Ultimate Ninja / \n",
"7-Musician / 36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / 116-Pokémon / 592-Synthesia / \n",
"5-Dance / \n",
"5-Dance / 49-School / 191-Cheerleading / 306-High school / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 431-Call of Duty: Advanced Warfare / \n",
"1061-Marriage proposal / \n",
"13-Musical ensemble / 14-Guitar / 24-String instrument / 1533-Lead guitar / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"14-Guitar / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 336-Scooter / 1448-Vespa / \n",
"12-Food / 227-Water / \n",
"419-Earth / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 38-Orchestra / \n",
"75-Wrestling / 641-Mask / \n",
"12-Food / 26-Cooking / 179-Kitchen / \n",
"1-Vehicle / 87-Boat / 308-Ship / \n",
"3-Concert / 16-Performance art / \n",
"9-Music video / 110-Album / \n",
"0-Games / 2-Video game / 51-Strategy video game / 171-Warcraft / 382-Dota 2 / 549-Defense of the Ancients / 1378-Multiplayer online battle arena / \n",
"2767-Captain Hook / \n",
"402-Jewellery / 859-Beadwork / 1300-Earrings / \n",
"1-Vehicle / 59-Winter sport / 84-Snow / 610-Snowmobile / 1201-Ski-Doo / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"12-Food / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 135-Drifting / \n",
"6-Animation / 25-Toy / 200-Doll / 439-Barbie / \n",
"1-Vehicle / 84-Snow / 120-Winter / 610-Snowmobile / \n",
"1007-Portal / \n",
"66-Bollywood / \n",
"1057-Embroidery / 1451-Ribbon / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 1447-Skylanders / 2321-Skylanders: Spyro's Adventure / \n",
"9-Music video / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"9-Music video / 541-Black-and-white / 3154-Pajamas / \n",
"8-Football / 54-Highlight film / \n",
"0-Games / 837-Roblox / \n",
"574-Elevator / \n",
"979-Star Trek / 2981-Spock / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 169-Rallying / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / \n",
"163-Running / \n",
"9-Music video / \n",
"726-Clarinet / \n",
"67-Cycling / 69-Bicycle / 147-River / \n",
"6-Animation / 20-Trailer / 159-World of Warcraft / 171-Warcraft / \n",
"3-Concert / \n",
"0-Games / \n",
"0-Games / 2-Video game / 51-Strategy video game / 165-League of Legends / 356-Star / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 378-Drag racing / 482-Asphalt / 552-Honda Civic / \n",
"0-Games / 13-Musical ensemble / 79-American football / 209-University / \n",
"6-Animation / 15-Cartoon / 183-Manga / \n",
"223-Weather / 490-Rain / \n",
"7-Musician / 14-Guitar / 24-String instrument / \n",
"0-Games / 8-Football / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / 318-Church / 1868-Preacher / \n",
"1-Vehicle / 4-Car / 88-Machine / \n",
"0-Games / 6-Animation / 19-PC game / \n",
"15-Cartoon / 61-Art / 97-Drawing / \n",
"7-Musician / 13-Musical ensemble / 14-Guitar / \n",
"18-Outdoor recreation / 59-Winter sport / 112-Ice skating / 312-Ice rink / 316-Figure skating / 632-Ice dancing / \n",
"992-Rocket / 3117-Match / \n",
"47-Personal computer / 145-Tablet computer / 189-iPad / \n",
"0-Games / 19-PC game / 1983-Infestation: Survivor Stories / \n",
"86-Plant / 136-Gardening / 1023-Rose / 2316-Floral design / 3177-Tulip / \n",
"0-Games / 8-Football / 65-Kick / 81-Athlete / \n",
"6-Animation / \n",
"31-Disc jockey / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 227-Water / 557-Salad / \n",
"2-Video game / 55-Video game console / 111-PlayStation 3 / 145-Tablet computer / 3238-GameStop / \n",
"3200-Morgana / \n",
"28-Fashion / 390-Model / 495-Runway / \n",
"512-Printing / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 73-Ball / \n",
"6-Animation / \n",
"3-Concert / 126-The Walt Disney Company / 144-Amusement park / \n",
"1-Vehicle / 25-Toy / 62-Train / 140-LEGO / 507-Thomas the Tank Engine / \n",
"3-Concert / 28-Fashion / 45-Cosmetics / 153-Dress / 206-Shoe / \n",
"13-Musical ensemble / 98-Festival / 253-Carnival / \n",
"0-Games / 2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / 178-Samsung Galaxy / \n",
"1-Vehicle / 4-Car / 70-Driving / 94-Dashcam / \n",
"0-Games / 2-Video game / 19-PC game / 20-Trailer / 43-Call of Duty / 55-Video game console / 138-Call of Duty: Black Ops II / 422-Call of Duty: Ghosts / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 64-Transport / 969-Volkswagen Beetle / \n",
"122-Weight training / \n",
"2-Video game / 8-Football / 90-Sports game / 127-Xbox 360 / 181-Xbox / 1199-Kinect / \n",
"78-Wedding / \n",
"9-Music video / \n",
"5-Dance / 49-School / \n",
"31-Disc jockey / 85-Combat / 1623-Shaolin Kung Fu / \n",
"9-Music video / \n",
"10-Animal / 166-Cat / 1347-Coffeehouse / \n",
"28-Fashion / 45-Cosmetics / 56-Hair / 195-Mascara / 235-Lipstick / 291-Rouge / 436-Concealer / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"3-Concert / 356-Star / \n",
"0-Games / 2-Video game / 17-Racing / \n",
"0-Games / 35-Minecraft / \n",
"1684-Hillsong Church / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 178-Samsung Galaxy / 487-Samsung Electronics / 1015-Samsung Galaxy Note series / 1727-Samsung Galaxy Note 3 / \n",
"3-Concert / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 1031-Fisheye lens / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 348-Fruit / 1175-Soap / \n",
"10-Animal / 48-Pet / 166-Cat / 182-Puppy / \n",
"35-Minecraft / 141-Microsoft Windows / \n",
"1143-Marimba / \n",
"232-Cake / \n",
"5-Dance / \n",
"75-Wrestling / 85-Combat / 117-Boxing / 399-Kickboxing / \n",
"78-Wedding / \n",
"1-Vehicle / 4-Car / 74-Truck / 1242-Car wash / \n",
"0-Games / 241-Fighting game / 486-M.U.G.E.N / 1506-Akuma / \n",
"5-Dance / 95-Talent show / \n",
"1-Vehicle / 4-Car / 158-Tire / 167-Four-wheel drive / 198-Off-road vehicle / 277-Sport utility vehicle / 901-Mitsubishi / 2432-Mitsubishi Pajero / \n",
"147-River / \n",
"8-Football / 73-Ball / \n",
"415-Sewing / 546-Stitch / 2961-Button / \n",
"1-Vehicle / 4-Car / 2583-Limousine / \n",
"5-Dance / \n",
"6-Animation / 20-Trailer / 123-Naruto / \n",
"353-Wall / 1350-Plaster / 2164-Silk / \n",
"0-Games / 1876-Beatmania IIDX / 2063-Beatmania / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 6-Animation / 15-Cartoon / \n",
"75-Wrestling / \n",
"1-Vehicle / 4-Car / 154-Wheel / 299-Volkswagen Passenger Cars / 493-Rim / \n",
"3-Concert / \n",
"59-Winter sport / 112-Ice skating / 262-Ice / 905-Flash Video / \n",
"12-Food / 226-Dessert / \n",
"439-Barbie / \n",
"14-Guitar / 24-String instrument / \n",
"61-Art / 457-Knitting / 476-Textile / 546-Stitch / \n",
"6-Animation / 89-Comics / \n",
"1-Vehicle / 4-Car / 533-Oil / 2303-Motor oil / \n",
"42-Fishing / 573-Paragliding / \n",
"8-Football / 21-Stadium / \n",
"1604-Satan / 1868-Preacher / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 76-Ball / 81-Athlete / \n",
"99-Christmas / \n",
"1-Vehicle / 4-Car / 3896-Mitsubishi Outlander / \n",
"0-Games / 1492-Persona / \n",
"0-Games / 138-Call of Duty: Black Ops II / 194-Call of Duty: Modern Warfare 2 / \n",
"0-Games / 2-Video game / 6-Animation / 240-Arcade game / 514-Tekken / 797-Tekken / \n",
"0-Games / 2-Video game / 20-Trailer / 34-Action-adventure game / 55-Video game console / 127-Xbox 360 / 2863-Max Payne 3 / \n",
"0-Games / 2-Video game / 1412-Half-Life / \n",
"126-The Walt Disney Company / \n",
"3-Concert / \n",
"5-Dance / 191-Cheerleading / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"955-Police officer / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 158-Tire / 207-Exhaust system / 947-Muffler / 1321-Škoda Auto / 2690-Škoda Octavia / \n",
"42-Fishing / 125-Television / 844-Bass / \n",
"0-Games / 8-Football / 90-Sports game / 180-FIFA 15 / 480-FIFA 13 / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 247-Stallion / 289-Dressage / 404-Livestock / 438-Mare / \n",
"10-Animal / 709-Bull / \n",
"67-Cycling / 69-Bicycle / \n",
"56-Hair / 106-Hairstyle / 340-Afro-textured hair / \n",
"9-Music video / \n",
"31-Disc jockey / 133-Nightclub / \n",
"5-Dance / \n",
"18-Outdoor recreation / 283-Rock / 548-Parachuting / 669-Extreme sport / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 89-Comics / 185-Dragon Ball / 244-Goku / 418-Gohan / 869-Dragon Ball Z: Budokai Tenkaichi / 876-Freeza / \n",
"121-Photography / 155-Camera / 566-Digital camera / 629-Camera lens / 721-Digital SLR / \n",
"9-Music video / \n",
"49-School / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 6-Animation / 34-Action-adventure game / 116-Pokémon / 186-Pokémon / 1387-Pokémon Battle Revolution / \n",
"2337-Passport / \n",
"10-Animal / 12-Food / 88-Machine / 3571-Lollipop / \n",
"28-Fashion / 696-T-shirt / 793-Shirt / 1525-Wallet / \n",
"5-Dance / 365-Fireworks / 440-Lion / \n",
"9-Music video / \n",
"0-Games / 8-Football / \n",
"46-Choir / \n",
"2-Video game / 37-Gadget / 55-Video game console / 111-PlayStation 3 / 127-Xbox 360 / 181-Xbox / 317-PlayStation / 677-Game controller / \n",
"12-Food / \n",
"326-Metin2 / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"5-Dance / 18-Outdoor recreation / 59-Winter sport / 112-Ice skating / 262-Ice / 316-Figure skating / 632-Ice dancing / \n",
"110-Album / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 74-Truck / 124-Tractor / 252-Tractor pulling / \n",
"9-Music video / \n",
"12-Food / 18-Outdoor recreation / 22-Nature / 86-Plant / 137-Tree / 208-Wood / 213-Forest / 673-Saw / 1073-Chainsaw / \n",
"0-Games / 2-Video game / 362-PlayStation Portable / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 19-PC game / 118-Grand Theft Auto V / 176-PlayStation 4 / 1275-Watch Dogs / \n",
"5-Dance / 85-Combat / \n",
"0-Games / 116-Pokémon / 186-Pokémon / \n",
"0-Games / 2-Video game / 739-Guild Wars / 839-Guild Wars 2 / 1130-Armour / \n",
"24-String instrument / 670-Banjo / \n",
"1042-Jumbotron / \n",
"0-Games / 59-Winter sport / 81-Athlete / 112-Ice skating / 142-Hockey / 312-Ice rink / \n",
"1-Vehicle / 42-Fishing / 87-Boat / \n",
"1-Vehicle / 10-Animal / 197-Trail / \n",
"9-Music video / \n",
"9-Music video / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 771-Noodle / \n",
"44-Drums / 227-Water / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 158-Tire / 198-Off-road vehicle / 360-Jeep / 1205-Jeep Wrangler / \n",
"3-Concert / 16-Performance art / \n",
"13-Musical ensemble / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / \n",
"1-Vehicle / 4-Car / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 2083-Border Collie / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 40-Road / \n",
"31-Disc jockey / \n",
"0-Games / 8-Football / 90-Sports game / 180-FIFA 15 / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 230-Brass instrument / 427-Trumpet / 634-Trombone / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / 923-Boeing 777 / 2391-American Airlines / 3705-Singapore Airlines / \n",
"22-Nature / 694-Tai chi / \n",
"10-Animal / 48-Pet / 132-Bird / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 89-Comics / 185-Dragon Ball / 244-Goku / 418-Gohan / 876-Freeza / 2144-Dragon Ball: Raging Blast 2 / \n",
"9-Music video / \n",
"1989-Oculus Rift / \n",
"0-Games / 2-Video game / 60-Basketball / 176-PlayStation 4 / 314-Xbox One / 991-NBA 2K15 / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / 993-Quarterback / 3987-Tight end / \n",
"164-Beach / 423-Coast / \n",
"10-Animal / 141-Microsoft Windows / \n",
"25-Toy / 140-LEGO / 264-Star Wars / 852-Lego Star Wars / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 77-Arena / 81-Athlete / \n",
"42-Fishing / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 135-Drifting / \n",
"38-Orchestra / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / \n",
"95-Talent show / 130-Gymnastics / \n",
"0-Games / 2-Video game / 366-Dragon / 420-The Elder Scrolls V: Skyrim / 441-The Elder Scrolls / \n",
"1-Vehicle / 18-Outdoor recreation / 40-Road / 67-Cycling / 69-Bicycle / 197-Trail / 220-Mountain bike / 284-Mountain biking / 520-Bicycle frame / 645-Road bicycle / \n",
"5-Dance / 13-Musical ensemble / \n",
"315-Knife / \n",
"5-Dance / \n",
"1-Vehicle / 4-Car / 94-Dashcam / \n",
"2-Video game / 237-Final Fantasy / \n",
"6-Animation / 25-Toy / 200-Doll / \n",
"10-Animal / 86-Plant / 113-House / 280-Wildlife / \n",
"402-Jewellery / 1010-Ring / 1089-Wire / \n",
"31-Disc jockey / \n",
"0-Games / 411-MapleStory / \n",
"6-Animation / 15-Cartoon / 89-Comics / 750-Sailor Moon / 1342-Sailor Moon / \n",
"18-Outdoor recreation / 22-Nature / 164-Beach / 233-Ocean / 423-Coast / \n",
"0-Games / 8-Football / 21-Stadium / 77-Arena / \n",
"117-Boxing / \n",
"1-Vehicle / 4-Car / 125-Television / 205-Newscaster / \n",
"920-Chair / 1454-Stairs / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / \n",
"6-Animation / 15-Cartoon / 126-The Walt Disney Company / 1139-Belle / 1265-Kyle Kingson / \n",
"6-Animation / 61-Art / 97-Drawing / 148-Painting / 408-Sketch / 528-Pencil / 628-Portrait / \n",
"8-Football / \n",
"6-Animation / \n",
"0-Games / 2-Video game / 241-Fighting game / 1291-Mortal Kombat / \n",
"1-Vehicle / 4-Car / 118-Grand Theft Auto V / 127-Xbox 360 / 176-PlayStation 4 / 181-Xbox / 314-Xbox One / 330-Money / 2212-Metal Gear Solid V: The Phantom Pain / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 85-Combat / 111-PlayStation 3 / 380-Assassin's Creed / 1435-Ezio Auditore da Firenze / 1485-Assassin's Creed: Brotherhood / \n",
"79-American football / \n",
"0-Games / 2-Video game / 55-Video game console / 366-Dragon / 1559-Spyro / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 2-Video game / 6-Animation / 241-Fighting game / 514-Tekken / 797-Tekken / 1470-Tekken Tag Tournament 2 / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / \n",
"0-Games / 2-Video game / 51-Strategy video game / 165-League of Legends / \n",
"0-Games / 2-Video game / 2379-Tom Clancy's Ghost Recon / \n",
"1-Vehicle / 124-Tractor / 249-Heavy equipment / \n",
"85-Combat / 117-Boxing / 399-Kickboxing / 699-Kick / \n",
"10-Animal / 48-Pet / 71-Dog / 162-Medicine / 1845-Pug / \n",
"14-Guitar / 24-String instrument / \n",
"0-Games / 2-Video game / 55-Video game console / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / 961-Sega Genesis / 1268-Tails / 1475-Knuckles the Echidna / 2327-Sonic the Hedgehog / \n",
"0-Games / 51-Strategy video game / 603-StarCraft II: Wings of Liberty / 2353-Heroes of the Storm / 2647-StarCraft II: Heart of the Swarm / 4353-Sarah Kerrigan / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / \n",
"0-Games / 2-Video game / 55-Video game console / \n",
"3-Concert / \n",
"10-Animal / 22-Nature / 280-Wildlife / 4253-Koala / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 165-League of Legends / \n",
"121-Photography / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"1-Vehicle / 22-Nature / 86-Plant / 136-Gardening / \n",
"93-Comedy / \n",
"0-Games / 54-Highlight film / 60-Basketball / 81-Athlete / 168-Basketball moves / 255-Slam dunk / \n",
"66-Bollywood / \n",
"20-Trailer / 1265-Kyle Kingson / 4301-Beauty and the Beast / \n",
"0-Games / 33-Weapon / 1938-Rust / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 719-Boeing 747 / 856-Airbus A320 family / 1316-Airbus A330 / 3671-Instrument landing system / \n",
"0-Games / 2-Video game / 20-Trailer / 111-PlayStation 3 / 127-Xbox 360 / 366-Dragon / 2621-GameTrailers / 3675-Dragon's Dogma / \n",
"293-Slide show / \n",
"9-Music video / \n",
"6-Animation / 18-Outdoor recreation / 22-Nature / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 40-Road / 158-Tire / \n",
"12-Food / 32-Recipe / 52-Dish / 215-Eating / 557-Salad / \n",
"3-Concert / 7-Musician / 24-String instrument / \n",
"1-Vehicle / 4-Car / 40-Road / 64-Transport / 74-Truck / 114-Engine / 259-Fire / 371-Firefighter / 796-Fire engine / 1020-Emergency vehicle / 1168-Neighbourhood / 1245-Alarm device / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 184-Vegetable / 210-Cookware and bakeware / 301-Meal / 557-Salad / 2614-Lettuce / \n",
"3-Concert / \n",
"6-Animation / 15-Cartoon / 185-Dragon Ball / 244-Goku / 418-Gohan / \n",
"162-Medicine / \n",
"0-Games / 2-Video game / 43-Call of Duty / 96-Soldier / 176-PlayStation 4 / 422-Call of Duty: Ghosts / \n",
"12-Food / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / \n",
"2-Video game / 1792-Router / \n",
"5-Dance / 16-Performance art / 430-Ballroom dance / \n",
"25-Toy / 323-Toddler / 354-Egg / 417-Play-Doh / 611-Kinder Surprise / \n",
"3-Concert / 7-Musician / 30-Drummer / 44-Drums / \n",
"12-Food / 867-Tea / \n",
"130-Gymnastics / 1417-Balance beam / \n",
"6-Animation / 15-Cartoon / \n",
"230-Brass instrument / 296-Saxophone / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"3-Concert / 5-Dance / \n",
"0-Games / 2-Video game / 55-Video game console / \n",
"0-Games / 8-Football / 21-Stadium / 76-Ball / \n",
"5-Dance / \n",
"1-Vehicle / 4-Car / 84-Snow / 120-Winter / 158-Tire / \n",
"8-Football / 28-Fashion / 206-Shoe / 333-Nike; Inc. / 1800-Football boot / \n",
"28-Fashion / 1178-Quest / \n",
"0-Games / 2-Video game / \n",
"7-Musician / 14-Guitar / \n",
"0-Games / 2-Video game / 14-Guitar / 445-Guitar Hero / \n",
"0-Games / 18-Outdoor recreation / 278-GoPro / \n",
"2-Video game / 55-Video game console / 176-PlayStation 4 / 181-Xbox / 282-Skin / 317-PlayStation / 677-Game controller / 2592-DualShock / 2769-Decal / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 94-Dashcam / 212-Highway / \n",
"162-Medicine / \n",
"0-Games / 2-Video game / 1630-Warframe / \n",
"61-Art / 97-Drawing / 148-Painting / 303-Paint / 873-Acrylic paint / 936-Watercolor paint / \n",
"12-Food / 88-Machine / 2196-Boiler / \n",
"5-Dance / 16-Performance art / 95-Talent show / 115-Ballet / \n",
"223-Weather / 502-Sky / 3942-Meteor shower / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / 842-Harp / 878-Composer / \n",
"0-Games / 8-Football / 59-Winter sport / 112-Ice skating / 142-Hockey / \n",
"402-Jewellery / 907-Necklace / \n",
"122-Weight training / 204-Gym / 214-Muscle / 541-Black-and-white / \n",
"93-Comedy / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 336-Scooter / 2686-Electric motorcycles and scooters / \n",
"18-Outdoor recreation / 721-Digital SLR / \n",
"7-Musician / 38-Orchestra / \n",
"31-Disc jockey / \n",
"752-Pasta / 2362-Porcelain / 3340-Cold porcelain / \n",
"7-Musician / 13-Musical ensemble / 14-Guitar / 39-Drums / 44-Drums / \n",
"1-Vehicle / 88-Machine / 124-Tractor / 249-Heavy equipment / \n",
"325-Vampire / \n",
"8-Football / 85-Combat / 358-Rugby football / \n",
"9-Music video / \n",
"22-Nature / 288-Underwater / 381-Underwater diving / 1135-Dolphin / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / 840-Museum / \n",
"9-Music video / \n",
"293-Slide show / \n",
"10-Animal / 48-Pet / 132-Bird / 598-Parrot / \n",
"882-Pen / \n",
"0-Games / 2-Video game / 445-Guitar Hero / \n",
"13-Musical ensemble / 1532-Military band / \n",
"0-Games / 2-Video game / 55-Video game console / 292-Handheld game console / 320-Super Smash Bros. / 782-Super Smash Bros. Melee / \n",
"468-Biology / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / \n",
"0-Games / 34-Action-adventure game / \n",
"1-Vehicle / 124-Tractor / 156-Agriculture / 249-Heavy equipment / 617-Maize / \n",
"309-Computer hardware / 516-Headphones / \n",
"0-Games / 2-Video game / 43-Call of Duty / 302-Call of Duty 4: Modern Warfare / \n",
"9-Music video / \n",
"66-Bollywood / \n",
"56-Hair / \n",
"31-Disc jockey / \n",
"31-Disc jockey / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"5-Dance / 356-Star / \n",
"6-Animation / 15-Cartoon / \n",
"32-Recipe / 162-Medicine / 3770-Toothpaste / \n",
"0-Games / 1138-Tower of Saviors / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 135-Drifting / 2785-Slot car / \n",
"7-Musician / 13-Musical ensemble / 203-Marching band / 1532-Military band / \n",
"6-Animation / \n",
"75-Wrestling / 2547-Staples Center / \n",
"136-Gardening / 428-Rail transport modelling / 1037-Grass / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 40-Road / 105-Motorcycling / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 189-iPad / 272-iPod / \n",
"6-Animation / 123-Naruto / \n",
"0-Games / 2-Video game / 33-Weapon / 834-Combat Arms / \n",
"20-Trailer / 1014-Movieclips / \n",
"0-Games / 2-Video game / 19-PC game / 159-World of Warcraft / 171-Warcraft / 1213-World of Warcraft: Wrath of the Lich King / 2139-Dungeon crawl / \n",
"10-Animal / 22-Nature / 48-Pet / 86-Plant / 136-Gardening / 166-Cat / 217-Hunting / 346-Kitten / 2924-Persian cat / \n",
"1-Vehicle / 4-Car / 124-Tractor / 156-Agriculture / 174-Farm / 617-Maize / 984-Silage / \n",
"3-Concert / 7-Musician / \n",
"10-Animal / 36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / 550-Dinosaur / \n",
"20-Trailer / 51-Strategy video game / 1072-Total War / 2810-Total War: Rome II / \n",
"1-Vehicle / 4-Car / 249-Heavy equipment / 1093-Loader / \n",
"31-Disc jockey / \n",
"6-Animation / 9-Music video / 15-Cartoon / 781-Pachinko / \n",
"3-Concert / \n",
"31-Disc jockey / 1240-Mambo / \n",
"5-Dance / 44-Drums / 3465-Davul / \n",
"0-Games / 59-Winter sport / 112-Ice skating / 142-Hockey / \n",
"14-Guitar / 24-String instrument / 1971-Epiphone / \n",
"5-Dance / \n",
"3-Concert / \n",
"5-Dance / \n",
"3-Concert / 6-Animation / 98-Festival / \n",
"0-Games / 6-Animation / 781-Pachinko / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 199-Wii / \n",
"5-Dance / \n",
"0-Games / 1-Vehicle / 2-Video game / 11-Motorsport / 199-Wii / 443-Wii U / 456-Go-kart / 518-Mario Kart / 1831-Mario Kart 8 / 4447-Rosalina / \n",
"0-Games / 8-Football / 21-Stadium / 358-Rugby football / \n",
"3380-Horror comedy / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 81-Athlete / \n",
"205-Newscaster / 330-Money / 2642-CNBC / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 64-Transport / \n",
"309-Computer hardware / 700-Hard disk drive / \n",
"2-Video game / 35-Minecraft / \n",
"20-Trailer / \n",
"6-Animation / 995-Attack on Titan / \n",
"110-Album / \n",
"0-Games / 2-Video game / 20-Trailer / 2277-The Lord of the Rings Online / \n",
"1-Vehicle / 4-Car / 691-Cargo / 707-Roof / 2355-Baggage / \n",
"3-Concert / \n",
"2-Video game / 17-Racing / 127-Xbox 360 / 181-Xbox / 386-Xbox / 677-Game controller / 705-Forza Motorsport / \n",
"487-Samsung Electronics / \n",
"22-Nature / 42-Fishing / 164-Beach / 222-Surfing / 223-Weather / 233-Ocean / 423-Coast / \n",
"1-Vehicle / 18-Outdoor recreation / \n",
"0-Games / 1-Vehicle / 124-Tractor / 156-Agriculture / 174-Farm / 617-Maize / 715-Farming Simulator / 984-Silage / \n",
"12-Food / 26-Cooking / 32-Recipe / 226-Dessert / 232-Cake / 236-Baking / 578-Cookie / 1024-Culinary art / 1230-Italian food / 1322-Biscuit / 3735-Tiramisu / \n",
"66-Bollywood / \n",
"66-Bollywood / 78-Wedding / \n",
"10-Animal / 48-Pet / 71-Dog / 179-Kitchen / 182-Puppy / 770-Banana / 1839-Rottweiler / \n",
"3-Concert / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"8-Football / 623-Coach / \n",
"0-Games / 2-Video game / 19-PC game / 1630-Warframe / \n",
"319-Robot / \n",
"1017-Yo-yo / \n",
"0-Games / 2-Video game / 19-PC game / 34-Action-adventure game / 3059-Dead Rising 2 / 3308-Dead Rising / \n",
"1-Vehicle / 4-Car / 41-Sports car / 234-Ford / 531-Hatchback / 1634-Ford Fiesta / \n",
"0-Games / 51-Strategy video game / 373-Clash of Clans / \n",
"42-Fishing / 164-Beach / 423-Coast / 3145-Spinnerbait / \n",
"46-Choir / \n",
"28-Fashion / 45-Cosmetics / 49-School / 4632-School supplies / \n",
"22-Nature / 1183-Karaoke box / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 84-Snow / 135-Drifting / 680-Subaru / 1286-Subaru / \n",
"12-Food / 442-Bread / \n",
"0-Games / 2-Video game / 33-Weapon / \n",
"12-Food / 795-Cupcake / \n",
"260-Prayer / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / 44-Drums / \n",
"6-Animation / 15-Cartoon / \n",
"10-Animal / \n",
"0-Games / 2-Video game / 55-Video game console / 413-PlayStation 2 / 4201-Jikkyō Powerful Pro Yakyū series / \n",
"66-Bollywood / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 40-Road / 118-Grand Theft Auto V / 297-Grand Theft Auto IV / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 81-Athlete / \n",
"0-Games / 60-Basketball / 168-Basketball moves / 1399-Streetball / \n",
"1-Vehicle / 2-Video game / 4-Car / \n",
"6-Animation / 15-Cartoon / 123-Naruto / 185-Dragon Ball / 396-One Piece / \n",
"3172-Lyre / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"8-Football / \n",
"0-Games / 51-Strategy video game / 165-League of Legends / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 47-Personal computer / 145-Tablet computer / 621-Asus / \n",
"42-Fishing / 147-River / 1542-Catfish / \n",
"0-Games / 8-Football / 358-Rugby football / \n",
"9-Music video / \n",
"209-University / \n",
"0-Games / 2-Video game / 3717-Suikoden / \n",
"59-Winter sport / 83-Skateboarding / 108-Skateboard / 281-Snowboarding / 4321-Mountainboarding / \n",
"162-Medicine / 475-Cheese / 600-Flour / 867-Tea / 1224-Cola / \n",
"6-Animation / 15-Cartoon / 31-Disc jockey / 290-Pony / 343-My Little Pony / \n",
"25-Toy / 140-LEGO / 4341-Anti-aircraft warfare / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"9-Music video / 78-Wedding / 153-Dress / 190-Bride / \n",
"9-Music video / \n",
"1-Vehicle / 336-Scooter / \n",
"59-Winter sport / 112-Ice skating / 142-Hockey / 259-Fire / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / \n",
"0-Games / \n",
"28-Fashion / 894-Jacket / 2188-Coat / \n",
"465-Microphone / \n",
"28-Fashion / 574-Elevator / 804-Shopping mall / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"25-Toy / \n",
"5-Dance / 6-Animation / 10-Animal / 15-Cartoon / 71-Dog / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 2-Video game / 6-Animation / 55-Video game console / 320-Super Smash Bros. / 697-Super Smash Bros. for Nintendo 3DS and Wii U / \n",
"0-Games / 85-Combat / 117-Boxing / \n",
"0-Games / 2-Video game / 20-Trailer / 55-Video game console / 111-PlayStation 3 / 127-Xbox 360 / 176-PlayStation 4 / 314-Xbox One / \n",
"93-Comedy / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"0-Games / 837-Roblox / \n",
"2-Video game / 47-Personal computer / 4549-Fruit Ninja / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"3-Concert / 7-Musician / 14-Guitar / 24-String instrument / \n",
"25-Toy / 3151-Kermit the Frog / \n",
"1-Vehicle / 25-Toy / 74-Truck / 140-LEGO / 822-Crane / \n",
"47-Personal computer / 131-Computer / 228-Laptop / 504-Web page / 525-Printer / \n",
"5-Dance / 16-Performance art / 95-Talent show / \n",
"47-Personal computer / 53-Keyboard / 309-Computer hardware / 513-Computer keyboard / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 87-Boat / 498-Motorboat / 2732-Offshore powerboat racing / \n",
"163-Running / 466-Gift / \n",
"9-Music video / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / \n",
"0-Games / 2-Video game / 10-Animal / 1116-Goat / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 111-PlayStation 3 / 118-Grand Theft Auto V / 127-Xbox 360 / 176-PlayStation 4 / 314-Xbox One / \n",
"492-Floor / \n",
"3-Concert / 46-Choir / 318-Church / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 152-Track / 161-Train station / \n",
"6-Animation / 15-Cartoon / \n",
"1-Vehicle / 50-Aircraft / 273-News program / \n",
"3-Concert / \n",
"47-Personal computer / 131-Computer / \n",
"0-Games / 2-Video game / 6-Animation / 514-Tekken / 797-Tekken / 1273-Tekken 6 / \n",
"0-Games / 54-Highlight film / 60-Basketball / 255-Slam dunk / \n",
"0-Games / 2-Video game / 1613-Crazyracing Kartrider / \n",
"25-Toy / 140-LEGO / 1007-Portal / 3597-GLaDOS / \n",
"1868-Preacher / \n",
"330-Money / \n",
"36-Piano / 53-Keyboard / \n",
"66-Bollywood / \n",
"1-Vehicle / 9-Music video / \n",
"162-Medicine / \n",
"0-Games / 2-Video game / 159-World of Warcraft / 171-Warcraft / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 1447-Skylanders / 2321-Skylanders: Spyro's Adventure / 2560-Skylanders: Swap Force / 2688-Skylanders: Giants / \n",
"20-Trailer / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 151-Landing / \n",
"238-Nail / 300-Nail art / 304-Nail polish / 307-Hand / 324-Finger / 338-Manicure / \n",
"10-Animal / 18-Outdoor recreation / 42-Fishing / 80-Horse / 287-Human swimming / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / 592-Synthesia / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / \n",
"130-Gymnastics / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 636-Rabbit / 824-Terrier / \n",
"10-Animal / 747-Retail / \n",
"0-Games / 411-MapleStory / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"12-Food / 227-Water / 499-Cloud / \n",
"93-Comedy / \n",
"126-The Walt Disney Company / \n",
"1-Vehicle / \n",
"1656-Projector / \n",
"60-Basketball / 1608-Injury / \n",
"0-Games / 2-Video game / 387-Spider-Man / \n",
"9-Music video / \n",
"45-Cosmetics / 56-Hair / 106-Hairstyle / 173-Eye shadow / 196-Eye / 218-Eyelash / \n",
"1-Vehicle / 4-Car / 41-Sports car / \n",
"2152-Selfie / \n",
"1-Vehicle / 9-Music video / 27-Motorcycle / 121-Photography / \n",
"3-Concert / \n",
"0-Games / 25-Toy / 455-Unidentified flying object / 1668-Claw crane / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 3229-Opel Kadett / \n",
"12-Food / 26-Cooking / 52-Dish / 369-Resort / 389-Restaurant / \n",
"843-The Doctor / \n",
"3-Concert / 7-Musician / 3489-Rock and Roll Hall of Fame / \n",
"641-Mask / 3240-Matryoshka doll / \n",
"59-Winter sport / 112-Ice skating / 142-Hockey / \n",
"1-Vehicle / 4-Car / 70-Driving / 121-Photography / 2248-Zoom lens / 2487-Canon PowerShot / \n",
"22-Nature / 998-Landscape / \n",
"122-Weight training / 204-Gym / 214-Muscle / 385-Face / 529-Squat / 532-Barbell / 1055-Back / \n",
"7-Musician / 24-String instrument / 150-Violin / 670-Banjo / \n",
"1-Vehicle / 25-Toy / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 266-Helicopter / 392-Radio-controlled helicopter / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"3-Concert / 95-Talent show / \n",
"126-The Walt Disney Company / 144-Amusement park / 605-Cruise ship / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"1-Vehicle / 4-Car / 74-Truck / 94-Dashcam / \n",
"415-Sewing / 525-Printer / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 74-Truck / 167-Four-wheel drive / 327-Mud / 565-Mud bogging / \n",
"66-Bollywood / \n",
"45-Cosmetics / 56-Hair / \n",
"93-Comedy / \n",
"0-Games / 2-Video game / 43-Call of Duty / 216-Call of Duty: Modern Warfare 3 / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 378-Drag racing / \n",
"121-Photography / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 74-Truck / \n",
"1-Vehicle / 4-Car / 64-Transport / 259-Fire / 371-Firefighter / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 1496-LittleBigPlanet 2 / 1672-LittleBigPlanet / \n",
"13-Musical ensemble / 30-Drummer / 44-Drums / 2376-Steelpan / \n",
"1-Vehicle / 2-Video game / 4-Car / 189-iPad / \n",
"9-Music video / \n",
"7-Musician / 14-Guitar / 24-String instrument / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 215-Eating / 301-Meal / 663-Street / 1438-Street food / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 594-Music festival / \n",
"12-Food / 86-Plant / 1720-Grape / 2473-Vineyard / \n",
"5-Dance / 16-Performance art / 59-Winter sport / 112-Ice skating / 316-Figure skating / 632-Ice dancing / \n",
"30-Drummer / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 422-Call of Duty: Ghosts / \n",
"2-Video game / 55-Video game console / 181-Xbox / 314-Xbox One / 386-Xbox / \n",
"0-Games / 2-Video game / 237-Final Fantasy / \n",
"102-Building / 998-Landscape / \n",
"8-Football / 20-Trailer / \n",
"3-Concert / 5-Dance / \n",
"1087-Donkey Kong / \n",
"0-Games / 60-Basketball / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 70-Driving / 277-Sport utility vehicle / 1916-Buick / \n",
"0-Games / 1-Vehicle / 2-Video game / 33-Weapon / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / 858-Carl Johnson / 3331-Quarry / \n",
"45-Cosmetics / 2183-Stardoll / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"47-Personal computer / 131-Computer / 228-Laptop / \n",
"22-Nature / 85-Combat / 86-Plant / \n",
"0-Games / 2-Video game / 20-Trailer / 159-World of Warcraft / 171-Warcraft / \n",
"4323-Cricket / \n",
"10-Animal / 48-Pet / 132-Bird / 2573-Domestic pigeon / \n",
"28-Fashion / 45-Cosmetics / 56-Hair / 193-Eye liner / 195-Mascara / 218-Eyelash / 235-Lipstick / \n",
"3-Concert / 7-Musician / 77-Arena / \n",
"14-Guitar / 100-Electric guitar / \n",
"322-Mixtape / 1721-Tetris / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"3-Concert / \n",
"83-Skateboarding / 108-Skateboard / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 78-Wedding / \n",
"28-Fashion / 153-Dress / \n",
"8-Football / \n",
"1-Vehicle / 62-Train / 119-Locomotive / 161-Train station / \n",
"20-Trailer / \n",
"8-Football / 21-Stadium / \n",
"273-News program / \n",
"1-Vehicle / 9-Music video / \n",
"122-Weight training / 204-Gym / 214-Muscle / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / \n",
"7-Musician / 36-Piano / 150-Violin / 401-Cello / 450-Viola / \n",
"0-Games / 35-Minecraft / 99-Christmas / \n",
"1-Vehicle / 4-Car / \n",
"78-Wedding / 190-Bride / 590-Wedding dress / \n",
"1-Vehicle / 40-Road / 94-Dashcam / 663-Street / \n",
"10-Animal / 42-Fishing / 91-Fish / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 24-String instrument / 38-Orchestra / 150-Violin / \n",
"7-Musician / 14-Guitar / 24-String instrument / 1231-Mandolin / \n",
"10-Animal / 15-Cartoon / 61-Art / 91-Fish / 97-Drawing / \n",
"0-Games / 8-Football / \n",
"9-Music video / \n",
"6-Animation / 1551-The Phantom of the Opera / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 728-Dirt track racing / \n",
"12-Food / 28-Fashion / 56-Hair / 106-Hairstyle / 340-Afro-textured hair / 857-Afro / \n",
"117-Boxing / \n",
"0-Games / 2-Video game / 19-PC game / 460-Yu-Gi-Oh! Trading Card Game / 494-Card game / \n",
"0-Games / 2-Video game / 23-Mobile phone / 37-Gadget / 55-Video game console / 159-World of Warcraft / 171-Warcraft / 189-iPad / \n",
"457-Knitting / 546-Stitch / 653-Thread / 1497-Wool / 1536-Woven fabric / 1712-Sweater / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 662-Pizza / \n",
"8-Football / 21-Stadium / 77-Arena / 1247-Penalty kick / \n",
"0-Games / 2-Video game / 35-Minecraft / 113-House / \n",
"1-Vehicle / 4-Car / 114-Engine / 449-Honda / 552-Honda Civic / \n",
"6-Animation / 15-Cartoon / 2339-Swan / \n",
"10-Animal / 164-Beach / 3123-Jellyfish / \n",
"31-Disc jockey / \n",
"1516-Belt / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 520-Bicycle frame / 645-Road bicycle / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 101-Telephone / 1009-iPhone 3G / 1032-BlackBerry / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 178-Samsung Galaxy / 487-Samsung Electronics / 1015-Samsung Galaxy Note series / 1349-USB flash drive / 1727-Samsung Galaxy Note 3 / \n",
"10-Animal / 48-Pet / 71-Dog / 4166-Rescue dog / \n",
"0-Games / 2-Video game / 1496-LittleBigPlanet 2 / \n",
"102-Building / 113-House / 170-Home improvement / 179-Kitchen / 580-Bedroom / 3716-Mobile home / \n",
"56-Hair / 1359-Hair iron / \n",
"60-Basketball / 3490-Eiffel Tower / \n",
"0-Games / 2-Video game / 116-Pokémon / 186-Pokémon / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 158-Tire / 220-Mountain bike / 659-Van / \n",
"59-Winter sport / 112-Ice skating / 316-Figure skating / 632-Ice dancing / \n",
"28-Fashion / 2114-Goth subculture / \n",
"12-Food / 102-Building / 208-Wood / 259-Fire / 279-Garden / 585-Lawn / 666-Concrete / 2258-Patio / \n",
"1-Vehicle / 18-Outdoor recreation / 22-Nature / 42-Fishing / 87-Boat / 147-River / 225-Lake / \n",
"12-Food / 720-Wine / 1720-Grape / \n",
"8-Football / \n",
"1-Vehicle / 208-Wood / 1294-Logging / 3073-Log splitter / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / \n",
"425-City / \n",
"3-Concert / 5-Dance / 16-Performance art / 1535-Prize / \n",
"419-Earth / 448-Planet / 774-Galaxy / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 768-Diamond / \n",
"376-Flute / 2375-Ney / \n",
"25-Toy / 126-The Walt Disney Company / 417-Play-Doh / \n",
"3-Concert / 7-Musician / 24-String instrument / 356-Star / \n",
"743-Edward Cullen / \n",
"8-Football / \n",
"1-Vehicle / 62-Train / 64-Transport / 94-Dashcam / 103-Rail transport / 152-Track / 161-Train station / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"6-Animation / \n",
"75-Wrestling / 777-WWE 2K / \n",
"31-Disc jockey / \n",
"23-Mobile phone / 29-Smartphone / 101-Telephone / 553-Sony Xperia / 3019-Sony Ericsson Xperia X10 / 3867-Sony Ericsson Xperia arc / 4635-Sony Ericsson Xperia X10 Mini / \n",
"1-Vehicle / 4-Car / 891-Mercedes-AMG / \n",
"1-Vehicle / 4-Car / 94-Dashcam / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 179-Kitchen / 210-Cookware and bakeware / \n",
"14-Guitar / 24-String instrument / 1754-Rocksmith / \n",
"5-Dance / 16-Performance art / 115-Ballet / \n",
"35-Minecraft / 242-Paper / 3410-Paper model / \n",
"3-Concert / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 30-Drummer / 39-Drums / 44-Drums / 1117-Camcorder / \n",
"46-Choir / \n",
"10-Animal / 48-Pet / 132-Bird / 598-Parrot / 1341-Parakeet / \n",
"242-Paper / 370-Origami / 1403-Flower bouquet / \n",
"5-Dance / \n",
"31-Disc jockey / 322-Mixtape / \n",
"9-Music video / 14-Guitar / \n",
"163-Running / \n",
"0-Games / 141-Microsoft Windows / \n",
"0-Games / 8-Football / 25-Toy / \n",
"36-Piano / 376-Flute / \n",
"12-Food / \n",
"1-Vehicle / 4-Car / 757-Boot / 1221-MINI / \n",
"293-Slide show / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 413-PlayStation 2 / 581-PlayStation / 3510-Ridge Racer / \n",
"6-Animation / \n",
"8-Football / 76-Ball / 905-Flash Video / \n",
"12-Food / 619-Manufacturing / 720-Wine / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 1380-Kawasaki motorcycles / 1951-Sport bike / 2494-Ninja ZX-6R / \n",
"12-Food / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"6-Animation / 4555-Amagami / \n",
"3-Concert / \n",
"10-Animal / 279-Garden / \n",
"5-Dance / 16-Performance art / 95-Talent show / \n",
"0-Games / 489-Board game / 799-Chess / 922-Tabletop game / \n",
"256-Tennis / 469-Bag / 974-Racket / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 188-Airline / 254-Runway / 472-Boeing 737 / \n",
"217-Hunting / 1716-Rat / \n",
"0-Games / 1-Vehicle / 4-Car / 34-Action-adventure game / 40-Road / 41-Sports car / 297-Grand Theft Auto IV / 2511-Pontiac / \n",
"113-House / 3779-Craigslist Inc. / \n",
"3-Concert / 68-Lighting / \n",
"1-Vehicle / 88-Machine / 124-Tractor / 156-Agriculture / \n",
"3-Concert / 5-Dance / 16-Performance art / 49-School / \n",
"6-Animation / 15-Cartoon / 325-Vampire / \n",
"1-Vehicle / 18-Outdoor recreation / 42-Fishing / 87-Boat / 114-Engine / 2338-Inflatable boat / \n",
"8-Football / \n",
"12-Food / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 94-Dashcam / \n",
"0-Games / 2-Video game / 111-PlayStation 3 / \n",
"18-Outdoor recreation / 42-Fishing / 147-River / 478-Fishing rod / 698-Fisherman / 1917-Fishing tackle / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 1301-Nissan Skyline / \n",
"6-Animation / 15-Cartoon / \n",
"10-Animal / 80-Horse / 290-Pony / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 33-Weapon / 1412-Half-Life / \n",
"0-Games / 240-Arcade game / 687-Taiko no Tatsujin / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / 265-Electronic keyboard / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / \n",
"0-Games / 2-Video game / 1027-Terraria / \n",
"15-Cartoon / 61-Art / 97-Drawing / 99-Christmas / 466-Gift / 528-Pencil / 538-Illustration / \n",
"93-Comedy / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 310-The Sims / 328-Simulation video game / 1553-The Sims 4 / 1713-The Sims / \n",
"0-Games / 2-Video game / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 139-Airport / 151-Landing / 175-Airliner / 188-Airline / 472-Boeing 737 / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 18-Outdoor recreation / 27-Motorcycle / 67-Cycling / 69-Bicycle / 172-Motocross / 1520-Minibike / \n",
"0-Games / 2-Video game / 33-Weapon / 834-Combat Arms / \n",
"31-Disc jockey / \n",
"75-Wrestling / 85-Combat / \n",
"347-Climbing / 828-Rock climbing / 1416-Cliff / \n",
"9-Music video / \n",
"36-Piano / 53-Keyboard / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 138-Call of Duty: Black Ops II / \n",
"12-Food / 26-Cooking / 32-Recipe / 475-Cheese / 534-Milk / \n",
"126-The Walt Disney Company / 144-Amusement park / 576-Walt Disney World / 1222-Fountain / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"0-Games / 1876-Beatmania IIDX / 2063-Beatmania / \n",
"287-Human swimming / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / \n",
"1-Vehicle / 4-Car / 41-Sports car / 70-Driving / 1763-Lotus Cars / \n",
"6-Animation / 15-Cartoon / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / 40-Road / 105-Motorcycling / 482-Asphalt / 608-Wheelie / \n",
"223-Weather / 419-Earth / 448-Planet / 593-Moon / 654-Sun / 1918-Comet / \n",
"0-Games / 2-Video game / 237-Final Fantasy / 1178-Quest / 2667-Final Fantasy IV / \n",
"5-Dance / 49-School / \n",
"766-President of the United States / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 179-Kitchen / 210-Cookware and bakeware / 226-Dessert / 236-Baking / 437-Cream / 968-Pie / 1208-Coconut / 1753-Mixer / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 20-Trailer / 40-Road / 41-Sports car / 135-Drifting / \n",
"18-Outdoor recreation / 163-Running / 327-Mud / \n",
"20-Trailer / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"0-Games / 8-Football / 54-Highlight film / 65-Kick / \n",
"28-Fashion / 78-Wedding / 153-Dress / 190-Bride / 590-Wedding dress / \n",
"5-Dance / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"3-Concert / 133-Nightclub / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 1431-Madison Square Garden / \n",
"18-Outdoor recreation / 163-Running / 674-Marathon / \n",
"365-Fireworks / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 1687-Heroes of Newerth / \n",
"1-Vehicle / 62-Train / 64-Transport / 161-Train station / \n",
"0-Games / 2-Video game / 35-Minecraft / 55-Video game console / 111-PlayStation 3 / 127-Xbox 360 / 176-PlayStation 4 / 181-Xbox / 317-PlayStation / 488-Door / \n",
"10-Animal / 709-Bull / 1373-Torero / \n",
"0-Games / 1187-Ragnarok Online / \n",
"92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 365-Fireworks / 393-Unmanned aerial vehicle / \n",
"0-Games / 2-Video game / \n",
"0-Games / 60-Basketball / 77-Arena / 168-Basketball moves / \n",
"0-Games / 2-Video game / 264-Star Wars / 714-Jedi / 1050-Star Wars: The Old Republic / 1882-Warrior / \n",
"22-Nature / 35-Minecraft / 1149-Castle / \n",
"3-Concert / 5-Dance / 227-Water / \n",
"25-Toy / 61-Art / 78-Wedding / 153-Dress / 590-Wedding dress / \n",
"6-Animation / 183-Manga / 325-Vampire / 2357-Vampire Knight / \n",
"3-Concert / \n",
"3-Concert / 16-Performance art / \n",
"1-Vehicle / 4-Car / 64-Transport / 74-Truck / 94-Dashcam / \n",
"60-Basketball / 85-Combat / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 135-Drifting / \n",
"3-Concert / 30-Drummer / 68-Lighting / \n",
"14-Guitar / 24-String instrument / 4165-Washburn Guitars / \n",
"86-Plant / 136-Gardening / 1460-Herb / 2795-Root / \n",
"3-Concert / 5-Dance / 68-Lighting / 263-Light / 842-Harp / 1375-Laser lighting display / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 269-Comic book / 335-Nintendo 3DS / 1092-Game Boy Advance / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 184-Vegetable / 301-Meal / 454-Sauce / 2187-Toast / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 96-Soldier / 3567-SOCOM / \n",
"0-Games / 2912-Flappy Bird / \n",
"3-Concert / 7-Musician / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 1003-Tuba / \n",
"6-Animation / 15-Cartoon / 434-Ninja / \n",
"12-Food / 26-Cooking / 49-School / 301-Meal / \n",
"28-Fashion / 2881-Wardrobe / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"9-Music video / 78-Wedding / 190-Bride / \n",
"0-Games / 2476-Maze / \n",
"20-Trailer / 223-Weather / \n",
"12-Food / 227-Water / 1329-Blender / \n",
"6-Animation / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 6-Animation / 514-Tekken / 2812-Kazuya Mishima / 3302-Tekken 5 / 3460-Tekken 5: Dark Resurrection / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / 197-Trail / 213-Forest / 363-Soil / \n",
"25-Toy / 89-Comics / 387-Spider-Man / 435-Action figure / 2368-Marvel Universe / \n",
"0-Games / 2-Video game / 1137-Crash Bandicoot / 3823-Crash Bandicoot: Warped / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 124-Tractor / 252-Tractor pulling / \n",
"25-Toy / 140-LEGO / 294-Batman / 884-Lego minifigure / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / \n",
"1-Vehicle / 4-Car / 270-Sedan / 332-Mercedes-Benz / 1310-Mercedes-Benz C-Class / 3826-Mercedes-Benz A-Class / \n",
"253-Carnival / \n",
"7-Musician / 14-Guitar / 24-String instrument / \n",
"99-Christmas / 558-Santa Claus / \n",
"3-Concert / \n",
"0-Games / 23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 189-iPad / 272-iPod / 331-iPod touch / \n",
"18-Outdoor recreation / 67-Cycling / 69-Bicycle / 197-Trail / 220-Mountain bike / 283-Rock / 284-Mountain biking / \n",
"1017-Yo-yo / \n",
"5-Dance / \n",
"46-Choir / \n",
"0-Games / 317-PlayStation / \n",
"9-Music video / \n",
"743-Edward Cullen / \n",
"6-Animation / 15-Cartoon / \n",
"12-Food / 52-Dish / \n",
"5-Dance / 59-Winter sport / 81-Athlete / 112-Ice skating / 316-Figure skating / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 70-Driving / 135-Drifting / 344-Coupé / 352-Need for Speed / 1600-Need for Speed: World / 2420-Audi TT / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / 1710-Pirates of the Caribbean / \n",
"1-Vehicle / 64-Transport / 74-Truck / 1611-Garbage truck / \n",
"0-Games / 2-Video game / 55-Video game console / 413-PlayStation 2 / 2142-Sega Saturn / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 229-Halo / 464-First-person Shooter / \n",
"12-Food / 32-Recipe / 354-Egg / 367-Drink / 534-Milk / 786-Juice / \n",
"2-Video game / 14-Guitar / 176-PlayStation 4 / \n",
"25-Toy / 68-Lighting / 140-LEGO / 263-Light / 1375-Laser lighting display / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / \n",
"55-Video game console / 414-Computer monitor / 1637-GameCube / 2166-Gamepad / \n",
"5-Dance / 49-School / 95-Talent show / 191-Cheerleading / 306-High school / \n",
"12-Food / 88-Machine / 812-Plastic / 2031-Molding / \n",
"3-Concert / 5-Dance / 16-Performance art / 38-Orchestra / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / \n",
"88-Machine / 400-Woodturning / 462-Table / \n",
"25-Toy / 89-Comics / 637-Five Nights at Freddy's / 1094-Puppet / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 105-Motorcycling / 336-Scooter / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 179-Kitchen / 239-Roasting / 391-Chicken meat / \n",
"1-Vehicle / 4-Car / 618-Fiat Automobiles / \n",
"0-Games / 8-Football / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / 265-Electronic keyboard / 446-Sheet music / 592-Synthesia / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"3-Concert / 7-Musician / 1659-Walmart / \n",
"1-Vehicle / 4-Car / 3353-Chevrolet Monte Carlo / \n",
"2-Video game / 55-Video game console / 677-Game controller / 2166-Gamepad / 2296-Joystick / \n",
"5-Dance / \n",
"6-Animation / \n",
"0-Games / 54-Highlight film / 142-Hockey / 1610-Field hockey / \n",
"54-Highlight film / 79-American football / \n",
"49-School / 657-Autumn / \n",
"1-Vehicle / 82-Airplane / \n",
"5-Dance / 79-American football / \n",
"0-Games / 59-Winter sport / 77-Arena / 112-Ice skating / 142-Hockey / 312-Ice rink / \n",
"49-School / 776-Kindergarten / \n",
"162-Medicine / \n",
"28-Fashion / 45-Cosmetics / 56-Hair / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"164-Beach / \n",
"162-Medicine / \n",
"365-Fireworks / \n",
"1-Vehicle / 87-Boat / 949-Rowing / 2678-Dragon boat / \n",
"36-Piano / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 141-Microsoft Windows / \n",
"14-Guitar / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 398-The Legend of Zelda / 1228-The Legend of Zelda: Ocarina of Time / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / 44-Drums / \n",
"6-Animation / 419-Earth / 448-Planet / 593-Moon / 712-Outer space / 1181-Mars / 1508-Spacecraft / 2942-Saturn / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"9-Music video / \n",
"60-Basketball / \n",
"14-Guitar / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 74-Truck / 1518-Chevrolet Silverado / \n",
"14-Guitar / \n",
"0-Games / 2-Video game / 19-PC game / 3840-The Binding of Isaac / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / 691-Cargo / 821-Rail freight transport / 2413-Union Pacific Railroad / \n",
"3-Concert / 7-Musician / \n",
"22-Nature / \n",
"1-Vehicle / 50-Aircraft / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 266-Helicopter / 392-Radio-controlled helicopter / \n",
"0-Games / 2-Video game / 326-Metin2 / \n",
"0-Games / 8-Football / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 135-Drifting / 705-Forza Motorsport / 790-Nissan GT-R / 1301-Nissan Skyline / 2105-Forza Horizon / 2247-Nissan Skyline GT-R / \n",
"18-Outdoor recreation / 22-Nature / 155-Camera / 1855-Video camera / \n",
"1369-Necktie / \n",
"1-Vehicle / 11-Motorsport / \n",
"5-Dance / 16-Performance art / \n",
"0-Games / 2-Video game / 90-Sports game / 180-FIFA 15 / \n",
"1-Vehicle / 42-Fishing / 87-Boat / \n",
"6-Animation / 15-Cartoon / 1164-Cartoon Network / 3478-Meme / \n",
"448-Planet / 593-Moon / 2756-Full moon / \n",
"0-Games / 2-Video game / 887-King Koopa / 1170-Super Mario World / \n",
"0-Games / 8-Football / 79-American football / \n",
"296-Saxophone / \n",
"20-Trailer / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 71-Dog / 1603-Fox / \n",
"122-Weight training / \n",
"12-Food / 86-Plant / 348-Fruit / \n",
"3-Concert / 68-Lighting / \n",
"37-Gadget / 47-Personal computer / 131-Computer / 145-Tablet computer / 228-Laptop / 414-Computer monitor / \n",
"18-Outdoor recreation / 365-Fireworks / \n",
"5-Dance / 16-Performance art / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 67-Cycling / 69-Bicycle / \n",
"3-Concert / 5-Dance / 98-Festival / \n",
"5-Dance / \n",
"5-Dance / 49-School / \n",
"1-Vehicle / 6-Animation / 25-Toy / 62-Train / 507-Thomas the Tank Engine / \n",
"12-Food / 342-Windows Media Video / 1088-Infomercial / \n",
"10-Animal / 166-Cat / \n",
"0-Games / 2-Video game / 34-Action-adventure game / \n",
"0-Games / 2-Video game / 43-Call of Duty / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / \n",
"7-Musician / 13-Musical ensemble / 98-Festival / \n",
"5-Dance / 9-Music video / \n",
"78-Wedding / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 149-Model aircraft / 223-Weather / 778-Gliding / 1263-Glider / 1317-Glider / \n",
"5-Dance / 9-Music video / 16-Performance art / \n",
"1601-Social media / \n",
"9-Music video / \n",
"85-Combat / 117-Boxing / \n",
"3-Concert / \n",
"7-Musician / 115-Ballet / \n",
"0-Games / 791-Magic: The Gathering / 1701-French fries / \n",
"259-Fire / 371-Firefighter / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 2-Video game / 1543-Crysis / \n",
"5-Dance / 115-Ballet / \n",
"28-Fashion / 45-Cosmetics / 835-Backpack / \n",
"46-Choir / \n",
"1726-Harlem Shake (meme) / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 111-PlayStation 3 / \n",
"28-Fashion / 206-Shoe / 333-Nike; Inc. / 368-Sneakers / 1423-Nike Air Max / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 235-Lipstick / 291-Rouge / \n",
"0-Games / 2-Video game / 35-Minecraft / 748-Iron Man / 4401-Iron Man's armor / \n",
"9-Music video / \n",
"14-Guitar / 24-String instrument / 2738-Fender Jazz Bass / \n",
"0-Games / 19-PC game / 35-Minecraft / 321-Roller coaster / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 157-Jet aircraft / 388-Fighter aircraft / 758-Air force / \n",
"14-Guitar / \n",
"6-Animation / 25-Toy / 816-Gundam / \n",
"8-Football / \n",
"28-Fashion / 153-Dress / 495-Runway / \n",
"136-Gardening / 1863-Fertilizer / \n",
"287-Human swimming / 1711-Synchronized swimming / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 73-Ball / \n",
"0-Games / 85-Combat / 412-Monster / \n",
"355-Comedian / \n",
"23-Mobile phone / 101-Telephone / \n",
"979-Star Trek / \n",
"467-Chipmunk / 846-Squirrel / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / \n",
"144-Amusement park / 287-Human swimming / 329-Swimming pool / 1053-Water park / 2360-Water polo / \n",
"12-Food / 25-Toy / 582-Candy / 3571-Lollipop / \n",
"3-Concert / 68-Lighting / \n",
"31-Disc jockey / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"0-Games / 2-Video game / 6-Animation / 20-Trailer / 664-CrossFire / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 89-Comics / 123-Naruto / 591-Naruto: Ultimate Ninja / \n",
"12-Food / 31-Disc jockey / 348-Fruit / 1193-Orange / \n",
"0-Games / 1-Vehicle / 2-Video game / 1613-Crazyracing Kartrider / \n",
"9-Music video / \n",
"126-The Walt Disney Company / \n",
"45-Cosmetics / 282-Skin / \n",
"298-Television advertisement / 337-Advertising / \n",
"121-Photography / 155-Camera / 721-Digital SLR / \n",
"31-Disc jockey / \n",
"42-Fishing / 469-Bag / \n",
"523-Cattle / \n",
"25-Toy / 242-Paper / 324-Finger / 370-Origami / 1094-Puppet / \n",
"8-Football / \n",
"93-Comedy / 364-Sitcom / \n",
"0-Games / 54-Highlight film / 60-Basketball / 168-Basketball moves / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 210-Cookware and bakeware / 239-Roasting / 354-Egg / 500-Rice / 1295-Stir frying / 2343-Caridean Shrimp / \n",
"0-Games / 2-Video game / 19-PC game / 159-World of Warcraft / 171-Warcraft / \n",
"920-Chair / \n",
"122-Weight training / \n",
"8-Football / 1016-Glove / \n",
"60-Basketball / \n",
"1-Vehicle / 4-Car / 67-Cycling / 69-Bicycle / 608-Wheelie / \n",
"366-Dragon / 756-Tibia / \n",
"7-Musician / 14-Guitar / 63-Acoustic guitar / \n",
"3304-Look-alike / \n",
"14-Guitar / \n",
"3-Concert / 14-Guitar / \n",
"3-Concert / 444-Family / \n",
"7-Musician / 13-Musical ensemble / 38-Orchestra / 150-Violin / 230-Brass instrument / \n",
"6-Animation / 15-Cartoon / 183-Manga / 708-Mermaid / 996-Princess / \n",
"3-Concert / \n",
"125-Television / 1348-Panasonic Corporation / 2786-3D television / \n",
"567-Compact disc / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 178-Samsung Galaxy / 487-Samsung Electronics / 1207-Samsung Galaxy S5 / 1525-Wallet / 3408-Samsung Galaxy S4 Mini / \n",
"3-Concert / \n",
"6-Animation / 25-Toy / 200-Doll / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / \n",
"25-Toy / 417-Play-Doh / 684-Ice cream / 723-Pig / \n",
"0-Games / 2-Video game / 25-Toy / 140-LEGO / 1302-The Lego Group / 1478-Lego Ninjago / \n",
"3-Concert / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 73-Ball / \n",
"12-Food / 26-Cooking / 32-Recipe / 226-Dessert / 232-Cake / 236-Baking / 442-Bread / 454-Sauce / 534-Milk / 560-Sugar / 1208-Coconut / 3003-Roll / 3728-Coconut milk / 4022-Bun / \n",
"0-Games / 35-Minecraft / \n",
"363-Soil / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"0-Games / 2-Video game / 1966-The Lord of the Rings / 2277-The Lord of the Rings Online / \n",
"68-Lighting / 263-Light / 1318-Flashlight / 3117-Match / 3851-Laser pointer / \n",
"1-Vehicle / 4-Car / 84-Snow / 459-Plough / 1548-Snowplow / \n",
"67-Cycling / 69-Bicycle / 570-Road bicycle racing / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / 1860-Sonic Adventure / \n",
"8-Football / \n",
"36-Piano / 53-Keyboard / \n",
"22-Nature / \n",
"9-Music video / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 172-Motocross / \n",
"0-Games / 2-Video game / 310-The Sims / 328-Simulation video game / 1553-The Sims 4 / 1713-The Sims / \n",
"6-Animation / 61-Art / 1206-Cinema 4D / \n",
"10-Animal / \n",
"25-Toy / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / \n",
"3-Concert / \n",
"1-Vehicle / 18-Outdoor recreation / 59-Winter sport / 84-Snow / 610-Snowmobile / 1210-Sled / \n",
"1-Vehicle / 87-Boat / 1514-Crab / \n",
"12-Food / 215-Eating / 301-Meal / 323-Toddler / \n",
"139-Airport / \n",
"1-Vehicle / 50-Aircraft / 151-Landing / 1992-Space Shuttle / \n",
"0-Games / 2-Video game / 1605-Second Life / \n",
"6-Animation / 15-Cartoon / 1133-Kirby / \n",
"0-Games / 51-Strategy video game / 791-Magic: The Gathering / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 96-Soldier / 834-Combat Arms / \n",
"67-Cycling / 69-Bicycle / 261-Tool / 1833-Hummer / \n",
"5-Dance / 95-Talent show / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"6-Animation / 15-Cartoon / 538-Illustration / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 172-Motocross / \n",
"22-Nature / 499-Cloud / 654-Sun / \n",
"0-Games / 2-Video game / 55-Video game console / 199-Wii / 1118-Kirby / 1133-Kirby / 4434-Kirby's Return to Dream Land / 4449-King Dedede / \n",
"14-Guitar / \n",
"12-Food / 992-Rocket / 1489-Coffeemaker / 1823-Espresso machine / 2057-Cappuccino / \n",
"1-Vehicle / 4-Car / 345-Loudspeaker / 484-Subwoofer / \n",
"75-Wrestling / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"6-Animation / 251-Sonic the Hedgehog / 538-Illustration / \n",
"23-Mobile phone / 29-Smartphone / 131-Computer / 1032-BlackBerry / \n",
"0-Games / \n",
"10-Animal / 20-Trailer / 48-Pet / 588-Reptile / 1444-Lizard / 3576-Iguana / \n",
"7-Musician / 9-Music video / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 49-School / 203-Marching band / 230-Brass instrument / \n",
"5-Dance / 16-Performance art / \n",
"89-Comics / 269-Comic book / 313-Book / \n",
"28-Fashion / 45-Cosmetics / 173-Eye shadow / 193-Eye liner / \n",
"88-Machine / 1751-Image scanner / \n",
"59-Winter sport / 177-Skiing / 305-Ski / \n",
"5-Dance / 9-Music video / \n",
"3-Concert / 16-Performance art / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / 2746-Twelve-string guitar / \n",
"20-Trailer / \n",
"0-Games / 60-Basketball / 168-Basketball moves / \n",
"2062-Kidney / \n",
"162-Medicine / 1375-Laser lighting display / \n",
"0-Games / 2-Video game / 19-PC game / 241-Fighting game / 486-M.U.G.E.N / 764-The King of Fighters / 2714-Iori Yagami / \n",
"10-Animal / 18-Outdoor recreation / 217-Hunting / \n",
"1-Vehicle / 4-Car / 59-Winter sport / 112-Ice skating / 142-Hockey / \n",
"23-Mobile phone / 429-Website / \n",
"298-Television advertisement / \n",
"322-Mixtape / \n",
"1-Vehicle / 11-Motorsport / 124-Tractor / 252-Tractor pulling / \n",
"5-Dance / 319-Robot / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / 207-Exhaust system / 1867-Honda CB600F / \n",
"1-Vehicle / 4-Car / 198-Off-road vehicle / 277-Sport utility vehicle / 3197-Citroën C4 / 3439-Compact car / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 230-Brass instrument / 1003-Tuba / \n",
"6-Animation / 593-Moon / 1744-Black Butler / \n",
"1-Vehicle / 4-Car / 12-Food / \n",
"6-Animation / 15-Cartoon / 185-Dragon Ball / 244-Goku / \n",
"361-Flamenco / \n",
"3-Concert / 7-Musician / 14-Guitar / 39-Drums / 44-Drums / \n",
"3-Concert / \n",
"1-Vehicle / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 377-BMX bike / \n",
"0-Games / 55-Video game console / 292-Handheld game console / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"5-Dance / 113-House / \n",
"0-Games / 60-Basketball / \n",
"144-Amusement park / 321-Roller coaster / 334-Amusement ride / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 42-Fishing / 91-Fish / 147-River / 902-Fishing bait / \n",
"0-Games / 51-Strategy video game / 165-League of Legends / 2004-Riot Games / \n",
"0-Games / 2-Video game / 8-Football / 21-Stadium / 79-American football / 90-Sports game / 176-PlayStation 4 / 314-Xbox One / 561-Madden NFL / 1813-Madden NFL 13 / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 118-Grand Theft Auto V / 1923-Niko Bellic / \n",
"13-Musical ensemble / 98-Festival / 203-Marching band / 250-Parade / 253-Carnival / \n",
"8-Football / 54-Highlight film / \n",
"5-Dance / \n",
"1-Vehicle / 4-Car / 3758-Toyota RAV4 / \n",
"359-Harry Potter (Literary Series) / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 229-Halo / 470-Halo 3 / 2240-Halo 5: Guardians / \n",
"0-Games / 2-Video game / 51-Strategy video game / 2841-Might and Magic / 3468-Heroes of Might and Magic / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 111-PlayStation 3 / 127-Xbox 360 / 176-PlayStation 4 / 314-Xbox One / 948-Far Cry / 1683-Far Cry 4 / 3618-Far Cry / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 39-Drums / 44-Drums / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 94-Dashcam / 212-Highway / 1593-Digital video recorder / \n",
"1-Vehicle / 27-Motorcycle / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / \n",
"12-Food / 26-Cooking / 32-Recipe / 179-Kitchen / 226-Dessert / 232-Cake / 236-Baking / 437-Cream / 475-Cheese / 1641-Cheesecake / 2964-Cream cheese / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 47-Personal computer / 961-Sega Genesis / \n",
"130-Gymnastics / 191-Cheerleading / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 179-Kitchen / 184-Vegetable / 210-Cookware and bakeware / 542-Potato / 557-Salad / \n",
"248-Hotel / \n",
"7-Musician / 36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"0-Games / 2-Video game / 25-Toy / 125-Television / 140-LEGO / 199-Wii / 264-Star Wars / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 40-Road / 482-Asphalt / \n",
"1-Vehicle / 4-Car / 114-Engine / 332-Mercedes-Benz / \n",
"1-Vehicle / 62-Train / 103-Rail transport / 119-Locomotive / 143-Railroad car / 428-Rail transport modelling / 1904-Electric locomotive / \n",
"6-Animation / 185-Dragon Ball / 244-Goku / \n",
"36-Piano / 53-Keyboard / 911-Studio / \n",
"3-Concert / \n",
"150-Violin / 3396-Ressha Sentai ToQger / \n",
"457-Knitting / 476-Textile / 546-Stitch / 653-Thread / \n",
"2425-Farmer / \n",
"5-Dance / 7-Musician / 14-Guitar / 16-Performance art / 24-String instrument / 361-Flamenco / \n",
"110-Album / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"372-Playing card / 746-Card manipulation / \n",
"1545-Floristry / \n",
"1-Vehicle / 27-Motorcycle / 3934-Radiator / 4495-Suzuki V-Strom 650 / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 297-Grand Theft Auto IV / 658-Grand Theft Auto: The Lost and Damned / \n",
"6-Animation / 15-Cartoon / 25-Toy / 200-Doll / 439-Barbie / 1441-Mattel / 2251-Ken / \n",
"0-Games / 8-Football / \n",
"117-Boxing / \n",
"45-Cosmetics / \n",
"96-Soldier / \n",
"1-Vehicle / 18-Outdoor recreation / 87-Boat / 225-Lake / 481-Kayak / 1090-Canoe / 1305-Canoeing / \n",
"5-Dance / 115-Ballet / \n",
"467-Chipmunk / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 380-Assassin's Creed / 1485-Assassin's Creed: Brotherhood / \n",
"10-Animal / 22-Nature / 706-Deer / \n",
"1-Vehicle / 4-Car / 327-Mud / \n",
"1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 169-Rallying / \n",
"9-Music video / 506-Garry's Mod / \n",
"243-Sonic the Hedgehog / 2849-Sonic Heroes / \n",
"20-Trailer / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 921-Grand Theft Auto: Vice City / \n",
"1-Vehicle / 87-Boat / \n",
"31-Disc jockey / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 47-Personal computer / 231-Supercar / 352-Need for Speed / 875-Need for Speed: Most Wanted / \n",
"1-Vehicle / 4-Car / 126-The Walt Disney Company / 144-Amusement park / 321-Roller coaster / 334-Amusement ride / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 64-Transport / 74-Truck / 286-Bus / \n",
"0-Games / 2-Video game / 33-Weapon / 1251-The Witcher / \n",
"8-Football / \n",
"0-Games / 1-Vehicle / 2-Video game / 11-Motorsport / 17-Racing / 27-Motorcycle / 105-Motorcycling / 111-PlayStation 3 / 127-Xbox 360 / 172-Motocross / 2930-MX vs. ATV / \n",
"93-Comedy / \n",
"0-Games / 1-Vehicle / 4-Car / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 172-Motocross / \n",
"827-Pump / \n",
"1001-Gemstone / \n",
"6-Animation / 15-Cartoon / 396-One Piece / \n",
"23-Mobile phone / 101-Telephone / \n",
"0-Games / 2-Video game / 19-PC game / 176-PlayStation 4 / 948-Far Cry / 1683-Far Cry 4 / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / \n",
"0-Games / 89-Comics / 1469-X-Men / \n",
"3-Concert / \n",
"45-Cosmetics / 56-Hair / 106-Hairstyle / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / \n",
"0-Games / 2-Video game / 43-Call of Duty / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / 181-Xbox / \n",
"6-Animation / 9-Music video / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 412-Monster / 3234-Kixeye; Inc. / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 165-League of Legends / \n",
"36-Piano / 53-Keyboard / 219-Accordion / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 178-Samsung Galaxy / 553-Sony Xperia / 1207-Samsung Galaxy S5 / 2906-Sony Xperia Z2 / \n",
"99-Christmas / 729-Christmas decoration / \n",
"264-Star Wars / \n",
"1-Vehicle / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 197-Trail / 220-Mountain bike / 284-Mountain biking / \n",
"6-Animation / 15-Cartoon / 1106-Character / \n",
"429-Website / 504-Web page / \n",
"0-Games / 8-Football / 21-Stadium / 77-Arena / \n",
"110-Album / 4686-Nisekoi / \n",
"3-Concert / \n",
"7-Musician / 230-Brass instrument / 296-Saxophone / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 184-Vegetable / 557-Salad / 2040-Korean food / 2637-Kimchi / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / 70-Driving / 105-Motorcycling / 1124-Yamaha YZF-R1 / 1380-Kawasaki motorcycles / 2494-Ninja ZX-6R / \n",
"0-Games / 2-Video game / \n",
"37-Gadget / 47-Personal computer / 131-Computer / 141-Microsoft Windows / 145-Tablet computer / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"9-Music video / 483-Costume / \n",
"6-Animation / 9-Music video / 10-Animal / 15-Cartoon / 970-Monkey / \n",
"0-Games / 2-Video game / 379-Touhou Project / \n",
"45-Cosmetics / 235-Lipstick / 1012-Lip gloss / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 2216-Honda Integra / 2800-Mazda RX-8 / \n",
"42-Fishing / 151-Landing / \n",
"3-Concert / \n",
"10-Animal / 48-Pet / 1116-Goat / 1198-Birth / \n",
"910-Dragon Quest / \n",
"6-Animation / 15-Cartoon / 877-Hatsune Miku: Project DIVA / \n",
"5-Dance / 16-Performance art / 59-Winter sport / 112-Ice skating / 262-Ice / 316-Figure skating / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 1-Vehicle / 50-Aircraft / 82-Airplane / 388-Fighter aircraft / \n",
"66-Bollywood / \n",
"10-Animal / 48-Pet / 166-Cat / 346-Kitten / \n",
"0-Games / 60-Basketball / 81-Athlete / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / \n",
"12-Food / 22-Nature / 86-Plant / 137-Tree / 1073-Chainsaw / \n",
"0-Games / 2-Video game / 89-Comics / 396-One Piece / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 76-Ball / 81-Athlete / \n",
"25-Toy / 1676-Aikatsu! / \n",
"0-Games / 6-Animation / 816-Gundam / 1564-Mobile Suit Gundam: Extreme Vs. / \n",
"5-Dance / \n",
"9-Music video / \n",
"141-Microsoft Windows / \n",
"1-Vehicle / 4-Car / 92-Radio-controlled model / 202-Radio-controlled car / \n",
"9-Music video / \n",
"6-Animation / 126-The Walt Disney Company / 3161-Tarzan / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 70-Driving / 270-Sedan / 552-Honda Civic / \n",
"1796-Skate / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"0-Games / 2-Video game / 116-Pokémon / 186-Pokémon / 1005-Pokémon Omega Ruby and Alpha Sapphire / \n",
"12-Food / 86-Plant / 137-Tree / 1460-Herb / \n",
"3-Concert / 16-Performance art / \n",
"342-Windows Media Video / 2447-Guzheng / \n",
"1-Vehicle / 4-Car / 360-Jeep / \n",
"0-Games / 2-Video game / 19-PC game / 241-Fighting game / 311-Street Fighter / 514-Tekken / 2812-Kazuya Mishima / \n",
"0-Games / 8-Football / 65-Kick / 73-Ball / 77-Arena / 90-Sports game / 180-FIFA 15 / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 135-Drifting / 378-Drag racing / 651-Stock car racing / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 70-Driving / 231-Supercar / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 3786-Killing Floor / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 102-Building / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / 472-Boeing 737 / 502-Sky / 2064-Boeing 737 Next Generation / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 139-Airport / 151-Landing / 502-Sky / 559-Night / 2136-Boeing 757 / 2174-Delta Air Lines / \n",
"5-Dance / 16-Performance art / 49-School / 153-Dress / 410-Teacher / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"23-Mobile phone / 29-Smartphone / 70-Driving / \n",
"0-Games / 81-Athlete / \n",
"3-Concert / \n",
"313-Book / 762-Angel / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"5-Dance / \n",
"97-Drawing / \n",
"4317-Japanese yen / \n",
"880-Trousers / \n",
"1-Vehicle / 205-Newscaster / \n",
"31-Disc jockey / \n",
"6-Animation / 15-Cartoon / 89-Comics / 1469-X-Men / \n",
"6-Animation / \n",
"0-Games / 2-Video game / 33-Weapon / 262-Ice / 1312-Dynasty Warriors / 4477-Dynasty Warriors Online / \n",
"5-Dance / 319-Robot / 483-Costume / 4438-Humanoid robot / \n",
"5-Dance / 95-Talent show / \n",
"1791-Stargate / \n",
"1-Vehicle / 4-Car / 299-Volkswagen Passenger Cars / 668-Volkswagen Golf / \n",
"12-Food / 18-Outdoor recreation / 86-Plant / 347-Climbing / 722-Waterfall / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 1032-BlackBerry / 3732-BlackBerry Bold / \n",
"12-Food / 2986-Dishwasher / \n",
"0-Games / 506-Garry's Mod / \n",
"14-Guitar / 516-Headphones / \n",
"1-Vehicle / 4-Car / \n",
"6-Animation / 25-Toy / 524-Littlest Pet Shop / \n",
"25-Toy / 35-Minecraft / 102-Building / 140-LEGO / 884-Lego minifigure / 3899-Lego Minecraft / \n",
"28-Fashion / 3169-Diva / \n",
"162-Medicine / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 111-PlayStation 3 / 127-Xbox 360 / 176-PlayStation 4 / 181-Xbox / 314-Xbox One / 1275-Watch Dogs / \n",
"147-River / 2412-Fate/stay night / \n",
"28-Fashion / 206-Shoe / 681-Leather / 757-Boot / \n",
"1-Vehicle / 4-Car / 41-Sports car / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / \n",
"1-Vehicle / 11-Motorsport / 286-Bus / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 456-Go-kart / 522-Helmet / \n",
"0-Games / 596-Monster Hunter / 1339-Monster Hunter Freedom Unite / 2994-Monster Hunter Freedom / \n",
"2-Video game / 37-Gadget / 47-Personal computer / 145-Tablet computer / \n",
"0-Games / 81-Athlete / 98-Festival / \n",
"12-Food / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"10-Animal / 80-Horse / 290-Pony / \n",
"3-Concert / \n",
"25-Toy / \n",
"1-Vehicle / 50-Aircraft / \n",
"0-Games / 51-Strategy video game / 373-Clash of Clans / 1950-The CW / \n",
"689-Harmonica / \n",
"0-Games / 6-Animation / 240-Arcade game / \n",
"1-Vehicle / 18-Outdoor recreation / 42-Fishing / 87-Boat / 225-Lake / 268-Recreational fishing / 510-Fishing lure / 844-Bass / 902-Fishing bait / \n",
"0-Games / 8-Football / 54-Highlight film / 65-Kick / \n",
"163-Running / \n",
"45-Cosmetics / 193-Eye liner / 196-Eye / 528-Pencil / \n",
"0-Games / 2-Video game / 90-Sports game / 180-FIFA 15 / 561-Madden NFL / \n",
"6-Animation / \n",
"164-Beach / 4596-Flying disc freestyle / \n",
"1-Vehicle / 4-Car / 332-Mercedes-Benz / 891-Mercedes-AMG / \n",
"1-Vehicle / 87-Boat / 579-Yacht / 4476-Shipyard / \n",
"6-Animation / \n",
"20-Trailer / \n",
"3-Concert / 16-Performance art / 68-Lighting / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / 1198-Birth / \n",
"5-Dance / 191-Cheerleading / \n",
"0-Games / 8-Football / 65-Kick / 699-Kick / 1623-Shaolin Kung Fu / \n",
"0-Games / 6-Animation / 366-Dragon / 460-Yu-Gi-Oh! Trading Card Game / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 74-Truck / 114-Engine / 167-Four-wheel drive / 197-Trail / 198-Off-road vehicle / 277-Sport utility vehicle / 327-Mud / 360-Jeep / 565-Mud bogging / \n",
"1-Vehicle / 4-Car / 319-Robot / 585-Lawn / 741-Mower / 805-Lawn mower / \n",
"86-Plant / 136-Gardening / 279-Garden / 906-Sowing / \n",
"5-Dance / \n",
"148-Painting / 408-Sketch / 415-Sewing / 476-Textile / 546-Stitch / 653-Thread / 997-Sewing machine / 1057-Embroidery / \n",
"10-Animal / 80-Horse / 438-Mare / 523-Cattle / 3212-Team roping / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 3059-Dead Rising 2 / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / \n",
"12-Food / 56-Hair / 106-Hairstyle / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / \n",
"155-Camera / 1013-Walking / \n",
"0-Games / 211-RuneScape / \n",
"28-Fashion / 676-Handbag / \n",
"10-Animal / 537-Zoo / \n",
"25-Toy / 323-Toddler / 3396-Ressha Sentai ToQger / \n",
"0-Games / 2-Video game / 1543-Crysis / \n",
"548-Parachuting / 765-Parachute / 1483-United States Air Force / \n",
"0-Games / 599-Dofus / \n",
"1-Vehicle / 4-Car / 70-Driving / 94-Dashcam / 245-BMW / 1607-BMW 3 Series (E36) / \n",
"89-Comics / 294-Batman / 1422-Statue / 2082-Catwoman / \n",
"49-School / \n",
"12-Food / \n",
"15-Cartoon / 61-Art / 394-Map / \n",
"1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 70-Driving / 135-Drifting / 231-Supercar / 705-Forza Motorsport / 790-Nissan GT-R / 1301-Nissan Skyline / \n",
"3-Concert / 350-Circus / 1894-Cannon / \n",
"12-Food / 86-Plant / 136-Gardening / 137-Tree / 673-Saw / 1073-Chainsaw / \n",
"9-Music video / \n",
"382-Dota 2 / 549-Defense of the Ancients / \n",
"9-Music video / \n",
"46-Choir / 49-School / \n",
"9-Music video / 61-Art / 408-Sketch / \n",
"10-Animal / 80-Horse / 709-Bull / \n",
"49-School / 54-Highlight film / 60-Basketball / 168-Basketball moves / \n",
"1-Vehicle / 4-Car / 92-Radio-controlled model / 167-Four-wheel drive / 202-Radio-controlled car / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 92-Radio-controlled model / 167-Four-wheel drive / 202-Radio-controlled car / 853-Dune buggy / \n",
"0-Games / 2-Video game / 441-The Elder Scrolls / 1130-Armour / 1410-The Elder Scrolls IV: Oblivion / \n",
"5-Dance / 7-Musician / 219-Accordion / \n",
"1-Vehicle / 50-Aircraft / \n",
"1-Vehicle / 245-BMW / \n",
"0-Games / 2-Video game / 257-Counter-Strike / \n",
"0-Games / 724-Rock Band / \n",
"0-Games / 60-Basketball / 168-Basketball moves / \n",
"1-Vehicle / 9-Music video / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 179-Kitchen / 542-Potato / 742-Countertop / 4415-Mashed potato / \n",
"1-Vehicle / 4-Car / 1502-Chrysler / \n",
"3-Concert / 7-Musician / 38-Orchestra / \n",
"3-Concert / \n",
"3575-Durarara!! / \n",
"0-Games / 2-Video game / 6-Animation / \n",
"458-Gold / \n",
"10-Animal / 48-Pet / 71-Dog / 1252-Dog agility / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / \n",
"1-Vehicle / 87-Boat / \n",
"10-Animal / 22-Nature / 213-Forest / 217-Hunting / 706-Deer / \n",
"3-Concert / 5-Dance / \n",
"9-Music video / \n",
"75-Wrestling / \n",
"3-Concert / \n",
"3-Concert / \n",
"144-Amusement park / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 74-Truck / 154-Wheel / 158-Tire / 167-Four-wheel drive / 198-Off-road vehicle / 234-Ford / 327-Mud / 565-Mud bogging / 2215-Ford Ranger / \n",
"0-Games / 2-Video game / 43-Call of Duty / 216-Call of Duty: Modern Warfare 3 / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 231-Supercar / 344-Coupé / 1040-Street racing / 1301-Nissan Skyline / 2216-Honda Integra / 3488-Fourth Generation Honda Integra / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 928-Borderlands 2 / \n",
"3-Concert / 5-Dance / \n",
"401-Cello / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / \n",
"8-Football / \n",
"6-Animation / 61-Art / 479-Graffiti / 1385-Street art / \n",
"1-Vehicle / 50-Aircraft / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 266-Helicopter / 392-Radio-controlled helicopter / \n",
"59-Winter sport / 112-Ice skating / 142-Hockey / 262-Ice / 312-Ice rink / \n",
"10-Animal / 132-Bird / 2339-Swan / \n",
"28-Fashion / 676-Handbag / \n",
"6-Animation / 15-Cartoon / \n",
"5-Dance / 115-Ballet / \n",
"25-Toy / 75-Wrestling / 311-Street Fighter / 435-Action figure / 563-Ryu / 749-Figurine / \n",
"170-Home improvement / \n",
"0-Games / 8-Football / 54-Highlight film / \n",
"12-Food / 342-Windows Media Video / 966-Sandwich / \n",
"219-Accordion / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 47-Personal computer / 145-Tablet computer / 1899-E-book / 2262-E-reader / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / \n",
"208-Wood / 261-Tool / 400-Woodturning / 1700-Lumber / \n",
"9-Music video / \n",
"10-Animal / 223-Weather / \n",
"99-Christmas / \n",
"8-Football / 90-Sports game / 453-Pro Evolution Soccer / \n",
"9-Music video / \n",
"46-Choir / 318-Church / \n",
"0-Games / 2-Video game / 553-Sony Xperia / 2770-Sony Xperia Z1 / 2882-Gladiator / \n",
"7-Musician / 13-Musical ensemble / 98-Festival / 203-Marching band / 250-Parade / \n",
"365-Fireworks / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 135-Drifting / 790-Nissan GT-R / 1810-Live for Speed / \n",
"3-Concert / 7-Musician / 14-Guitar / 2393-Didgeridoo / \n",
"164-Beach / 248-Hotel / 369-Resort / \n",
"1552-Video lesson / \n",
"12-Food / 86-Plant / 137-Tree / 1720-Grape / 2473-Vineyard / \n",
"0-Games / 2-Video game / 739-Guild Wars / 839-Guild Wars 2 / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"14-Guitar / \n",
"1-Vehicle / 92-Radio-controlled model / 202-Radio-controlled car / \n",
"5-Dance / \n",
"2410-Camel / \n",
"14-Guitar / 100-Electric guitar / \n",
"0-Games / \n",
"16-Performance art / 1139-Belle / 1265-Kyle Kingson / \n",
"452-Watch / 2168-G-Shock / \n",
"0-Games / 2-Video game / 25-Toy / 425-City / \n",
"0-Games / 2-Video game / 116-Pokémon / 186-Pokémon / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 30-Drummer / \n",
"28-Fashion / 2385-Closet / \n",
"3-Concert / 16-Performance art / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 40-Road / 64-Transport / 482-Asphalt / 1867-Honda CB600F / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 39-Drums / \n",
"3-Concert / \n",
"0-Games / 18-Outdoor recreation / 59-Winter sport / 112-Ice skating / 142-Hockey / 262-Ice / 312-Ice rink / 1973-Hockey puck / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 42-Fishing / 91-Fish / 147-River / 225-Lake / 268-Recreational fishing / 1018-Salmon / 2094-Rainbow trout / \n",
"22-Nature / 120-Winter / 339-Architecture / 998-Landscape / \n",
"61-Art / 479-Graffiti / \n",
"12-Food / 539-Home appliance / 1439-Refrigerator / \n",
"1-Vehicle / \n",
"5-Dance / 9-Music video / 466-Gift / \n",
"0-Games / 51-Strategy video game / 4539-Neighbours from Hell / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"0-Games / 51-Strategy video game / 165-League of Legends / \n",
"12-Food / 1175-Soap / 4325-Detergent / \n",
"1-Vehicle / 62-Train / 119-Locomotive / 794-Bridge / \n",
"0-Games / 530-Cue sports / 568-Pool / 994-Cue stick / 1364-Nine-ball / \n",
"5-Dance / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 247-Stallion / 289-Dressage / 351-Jumping / \n",
"8-Football / 76-Ball / \n",
"1-Vehicle / 4-Car / \n",
"18-Outdoor recreation / 22-Nature / 137-Tree / 213-Forest / 395-Camping / 914-Tent / \n",
"2-Video game / 6-Animation / \n",
"0-Games / 2-Video game / 240-Arcade game / \n",
"2-Video game / 47-Personal computer / 131-Computer / 228-Laptop / 414-Computer monitor / 3000-ThinkPad / \n",
"0-Games / 35-Minecraft / \n",
"3-Concert / 16-Performance art / \n",
"0-Games / 2-Video game / \n",
"56-Hair / 106-Hairstyle / \n",
"170-Home improvement / 179-Kitchen / 246-Furniture / 652-Bathroom / \n",
"273-News program / \n",
"85-Combat / 937-Black belt / 2822-Submission wrestling / \n",
"3-Concert / 5-Dance / \n",
"35-Minecraft / \n",
"1-Vehicle / 4-Car / 70-Driving / 733-Washing / 1242-Car wash / \n",
"3-Concert / 5-Dance / 68-Lighting / \n",
"1-Vehicle / 27-Motorcycle / 207-Exhaust system / 3592-Kawasaki Ninja 650R / \n",
"627-Clock / \n",
"1492-Persona / 2896-Shin Megami Tensei: Persona 3 / \n",
"3-Concert / 7-Musician / \n",
"345-Loudspeaker / 484-Subwoofer / \n",
"0-Games / 326-Metin2 / \n",
"110-Album / \n",
"2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 101-Telephone / 272-iPod / 331-iPod touch / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 40-Road / 212-Highway / 245-BMW / 800-Mountain pass / \n",
"1-Vehicle / 4-Car / 2991-Ferrari 599 GTB Fiorano / \n",
"6-Animation / 9-Music video / 15-Cartoon / 803-DC Comics / 2782-Starfire / \n",
"1-Vehicle / 4-Car / 403-Toyota / \n",
"1345-Dean Winchester / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 47-Personal computer / 72-iPhone / 131-Computer / 145-Tablet computer / 178-Samsung Galaxy / 487-Samsung Electronics / 1015-Samsung Galaxy Note series / 2416-Samsung Galaxy Tab 10.1 / 2802-Samsung Galaxy Note 10.1 / \n",
"13-Musical ensemble / 21-Stadium / 77-Arena / 79-American football / 203-Marching band / 209-University / \n",
"2-Video game / 1241-Assassin's Creed IV: Black Flag / 1858-Poster / 3238-GameStop / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 57-Race track / 595-Champion / 1609-Auto Race / \n",
"20-Trailer / \n",
"0-Games / 1-Vehicle / 124-Tractor / 156-Agriculture / 174-Farm / 715-Farming Simulator / \n",
"238-Nail / 300-Nail art / 304-Nail polish / 338-Manicure / 1128-Glitter / \n",
"0-Games / 302-Call of Duty 4: Modern Warfare / \n",
"20-Trailer / \n",
"0-Games / 731-Coin / 831-Plants vs. Zombies / 1374-Plants vs. Zombies 2: It's About Time / \n",
"12-Food / 348-Fruit / 780-Fat / \n",
"49-School / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / 2060-Roulette / \n",
"10-Animal / 48-Pet / 71-Dog / 84-Snow / 120-Winter / \n",
"0-Games / 19-PC game / 211-RuneScape / \n",
"7-Musician / \n",
"5-Dance / 2071-Indiana Jones / \n",
"860-Christian Church / \n",
"380-Assassin's Creed / 1241-Assassin's Creed IV: Black Flag / \n",
"12-Food / 125-Television / 3044-Sesame / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 55-Video game console / \n",
"0-Games / 60-Basketball / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / \n",
"2-Video game / 20-Trailer / \n",
"0-Games / 60-Basketball / 176-PlayStation 4 / 314-Xbox One / 991-NBA 2K15 / \n",
"0-Games / 2-Video game / 19-PC game / 2594-Mount & Blade: Warband / 2622-Mount & Blade / \n",
"10-Animal / 71-Dog / \n",
"1-Vehicle / 949-Rowing / 1090-Canoe / 1305-Canoeing / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 219-Accordion / \n",
"365-Fireworks / \n",
"1605-Second Life / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / 716-Coral / \n",
"8-Football / 3303-Table football / \n",
"425-City / \n",
"1-Vehicle / 62-Train / 64-Transport / 84-Snow / 103-Rail transport / 120-Winter / 152-Track / 161-Train station / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"10-Animal / 80-Horse / \n",
"0-Games / 90-Sports game / 180-FIFA 15 / \n",
"0-Games / 8-Football / 60-Basketball / \n",
"1-Vehicle / 22-Nature / 642-Leaf / 657-Autumn / \n",
"18-Outdoor recreation / 22-Nature / 395-Camping / 614-Hiking / 835-Backpack / 1013-Walking / 1203-Backpacking / \n",
"12-Food / 26-Cooking / 475-Cheese / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 1155-iPhone 3GS / \n",
"13-Musical ensemble / \n",
"1-Vehicle / 4-Car / 25-Toy / 92-Radio-controlled model / 202-Radio-controlled car / \n",
"0-Games / 2-Video game / 8-Football / 90-Sports game / 180-FIFA 15 / \n",
"1-Vehicle / 4-Car / 114-Engine / 449-Honda / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 382-Dota 2 / 549-Defense of the Ancients / \n",
"326-Metin2 / 342-Windows Media Video / \n",
"1-Vehicle / 62-Train / 64-Transport / 161-Train station / 224-Rapid transit / \n",
"0-Games / 23-Mobile phone / 366-Dragon / \n",
"12-Food / 26-Cooking / 58-Cuisine / 274-Meat / 589-Grilling / 620-Barbecue / \n",
"798-Shampoo / 920-Chair / 1271-Beauty salon / \n",
"516-Headphones / 832-Headset / \n",
"609-Tattoo / 1834-Mehndi / \n",
"99-Christmas / \n",
"3-Concert / 7-Musician / 38-Orchestra / \n",
"0-Games / 59-Winter sport / 112-Ice skating / 316-Figure skating / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 608-Wheelie / 4133-Yamaha DT125 / \n",
"0-Games / 2-Video game / 19-PC game / 739-Guild Wars / 839-Guild Wars 2 / \n",
"1-Vehicle / 4-Car / \n",
"8-Football / \n",
"6-Animation / 10-Animal / 15-Cartoon / 80-Horse / 2155-Unicorn / \n",
"0-Games / 1-Vehicle / 2-Video game / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"0-Games / 489-Board game / 799-Chess / 922-Tabletop game / \n",
"28-Fashion / 153-Dress / \n",
"5-Dance / 115-Ballet / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"0-Games / 43-Call of Duty / 194-Call of Duty: Modern Warfare 2 / \n",
"12-Food / \n",
"9-Music video / 61-Art / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"200-Doll / 1222-Fountain / 3605-Puppetry / \n",
"56-Hair / 106-Hairstyle / 340-Afro-textured hair / 798-Shampoo / 919-Hair conditioner / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / 923-Boeing 777 / \n",
"3-Concert / \n",
"260-Prayer / \n",
"5-Dance / 66-Bollywood / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"325-Vampire / \n",
"3-Concert / \n",
"10-Animal / 22-Nature / \n",
"1-Vehicle / 4-Car / 683-Renault / 2785-Slot car / \n",
"141-Microsoft Windows / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 297-Grand Theft Auto IV / 1595-Television set / \n",
"5-Dance / 16-Performance art / 99-Christmas / \n",
"0-Games / 2-Video game / 51-Strategy video game / 1570-Command & Conquer / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 89-Comics / 185-Dragon Ball / 244-Goku / 876-Freeza / \n",
"9-Music video / 1791-Stargate / \n",
"1-Vehicle / 11-Motorsport / 124-Tractor / 156-Agriculture / 249-Heavy equipment / 252-Tractor pulling / 459-Plough / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 231-Supercar / 705-Forza Motorsport / 952-Chevrolet Corvette / 1530-Forza Motorsport 4 / 2885-Chevrolet Corvette (C6) / 4135-Chevrolet Corvette C5 Z06 / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"28-Fashion / 56-Hair / 793-Shirt / \n",
"5-Dance / \n",
"6-Animation / \n",
"0-Games / 76-Ball / 81-Athlete / 256-Tennis / 974-Racket / 1337-Serve / \n",
"131-Computer / 141-Microsoft Windows / \n",
"402-Jewellery / 829-Bead / 859-Beadwork / 907-Necklace / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 77-Arena / 81-Athlete / \n",
"5-Dance / \n",
"25-Toy / 1063-Ink / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 91-Fish / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / 1316-Airbus A330 / \n",
"9-Music video / \n",
"5-Dance / \n",
"12-Food / 1408-Snack / \n",
"0-Games / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / \n",
"113-House / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 212-Highway / \n",
"843-The Doctor / \n",
"1-Vehicle / 50-Aircraft / 471-Wing / 778-Gliding / 1263-Glider / 1317-Glider / \n",
"139-Airport / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"110-Album / \n",
"3-Concert / \n",
"1-Vehicle / 62-Train / 64-Transport / 94-Dashcam / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / 224-Rapid transit / 821-Rail freight transport / \n",
"960-Badminton / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 70-Driving / 135-Drifting / 1575-Clutch / \n",
"6-Animation / 419-Earth / 701-Beyblade / \n",
"3-Concert / 16-Performance art / 68-Lighting / \n",
"10-Animal / 588-Reptile / \n",
"18-Outdoor recreation / 59-Winter sport / 84-Snow / 120-Winter / 281-Snowboarding / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 94-Dashcam / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 85-Combat / 4690-Shadow Fighter / \n",
"5-Dance / 49-School / 95-Talent show / 130-Gymnastics / 191-Cheerleading / \n",
"1-Vehicle / 4-Car / \n",
"8-Football / 54-Highlight film / 79-American football / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / 471-Wing / 502-Sky / 923-Boeing 777 / 1705-Boeing 767 / 2391-American Airlines / \n",
"322-Mixtape / \n",
"0-Games / 2-Video game / 19-PC game / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / 858-Carl Johnson / \n",
"25-Toy / 140-LEGO / 387-Spider-Man / \n",
"12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / 179-Kitchen / 1725-Custard / \n",
"18-Outdoor recreation / 395-Camping / 943-Stove / 1203-Backpacking / 2670-Portable stove / \n",
"0-Games / 2-Video game / 33-Weapon / 420-The Elder Scrolls V: Skyrim / 441-The Elder Scrolls / \n",
"1-Vehicle / 88-Machine / \n",
"2233-Derek Shepherd / \n",
"3-Concert / \n",
"7-Musician / 24-String instrument / 150-Violin / \n",
"0-Games / 2-Video game / 51-Strategy video game / 382-Dota 2 / 549-Defense of the Ancients / \n",
"122-Weight training / 204-Gym / 214-Muscle / 532-Barbell / 1250-Biceps / 2045-Biceps curl / \n",
"102-Building / 113-House / 170-Home improvement / 276-Room / 279-Garden / 339-Architecture / 2220-Shed / 2291-Log cabin / \n",
"6-Animation / 25-Toy / 200-Doll / \n",
"6-Animation / 185-Dragon Ball / 244-Goku / 1165-Majin Buu / \n",
"98-Festival / 253-Carnival / \n",
"0-Games / 2-Video game / 88-Machine / 240-Arcade game / 735-Kamen Rider Series / 2557-Kamen Rider Battle: Ganbaride / \n",
"69-Bicycle / 1067-Lock / 2516-Lock picking / 3061-Cylinder / \n",
"1-Vehicle / 27-Motorcycle / 522-Helmet / 1243-Mirror / 1903-Motorcycle helmet / \n",
"20-Trailer / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / \n",
"1-Vehicle / 11-Motorsport / 67-Cycling / 69-Bicycle / 220-Mountain bike / 377-BMX bike / \n",
"8-Football / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 20-Trailer / 55-Video game console / 111-PlayStation 3 / 127-Xbox 360 / 3406-Bayonetta / \n",
"420-The Elder Scrolls V: Skyrim / \n",
"8-Football / 90-Sports game / 453-Pro Evolution Soccer / \n",
"5-Dance / 16-Performance art / \n",
"78-Wedding / 190-Bride / \n",
"31-Disc jockey / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 85-Combat / 111-PlayStation 3 / 127-Xbox 360 / 434-Ninja / 2192-Ninja Gaiden / \n",
"3-Concert / 22-Nature / 98-Festival / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 92-Radio-controlled model / 202-Radio-controlled car / 327-Mud / 565-Mud bogging / \n",
"5-Dance / 16-Performance art / 38-Orchestra / \n",
"264-Star Wars / 4205-Motel / \n",
"31-Disc jockey / \n",
"376-Flute / 1326-Recorder / \n",
"0-Games / 2-Video game / 47-Personal computer / 228-Laptop / \n",
"0-Games / 411-MapleStory / 3910-Mackerel / \n",
"3-Concert / 5-Dance / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 175-Airliner / 188-Airline / 471-Wing / 923-Boeing 777 / 2587-Air France / \n",
"3-Concert / 16-Performance art / \n",
"3-Concert / 7-Musician / \n",
"3-Concert / \n",
"12-Food / 427-Trumpet / \n",
"10-Animal / 48-Pet / 71-Dog / 1084-Bulldog / \n",
"6-Animation / 15-Cartoon / 750-Sailor Moon / 1342-Sailor Moon / 2761-Venus / 4287-Mercury / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"0-Games / 2-Video game / 35-Minecraft / 2171-The Hunger Games / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 194-Call of Duty: Modern Warfare 2 / \n",
"12-Food / \n",
"0-Games / 2590-Mahjong / \n",
"1-Vehicle / 17-Racing / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 570-Road bicycle racing / 645-Road bicycle / \n",
"6-Animation / 3437-Maid Sama! / \n",
"1-Vehicle / 18-Outdoor recreation / 42-Fishing / 87-Boat / 225-Lake / 2836-Walleye / 3788-Crappie / \n",
"75-Wrestling / \n",
"102-Building / 113-House / 170-Home improvement / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 245-BMW / 874-BMW M3 / \n",
"6-Animation / 12-Food / 208-Wood / 246-Furniture / 462-Table / \n",
"10-Animal / 48-Pet / 71-Dog / 881-German Shepherd / 3116-Schutzhund / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 272-iPod / \n",
"1-Vehicle / 4-Car / 484-Subwoofer / 3086-Honda CR-V / \n",
"5-Dance / 16-Performance art / \n",
"0-Games / 51-Strategy video game / 241-Fighting game / 2320-Guilty Gear / \n",
"0-Games / 256-Tennis / 1626-Forehand / \n",
"113-House / 179-Kitchen / 276-Room / 1142-Laundry / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 98-Festival / \n",
"12-Food / 26-Cooking / 179-Kitchen / \n",
"10-Animal / 48-Pet / 71-Dog / 1839-Rottweiler / 2049-Police dog / 3116-Schutzhund / \n",
"326-Metin2 / \n",
"755-Hetalia: Axis Powers / \n",
"1-Vehicle / 50-Aircraft / 139-Airport / \n",
"3-Concert / 16-Performance art / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 218-Eyelash / 235-Lipstick / \n",
"7-Musician / 13-Musical ensemble / 230-Brass instrument / \n",
"0-Games / 2-Video game / 43-Call of Duty / 138-Call of Duty: Black Ops II / \n",
"0-Games / 2-Video game / 10-Animal / 132-Bird / 847-Angry Birds / 3319-Angry Birds / \n",
"3-Concert / 16-Performance art / 133-Nightclub / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / 1562-Viola caipira / \n",
"6-Animation / 15-Cartoon / \n",
"31-Disc jockey / \n",
"0-Games / 2-Video game / 10-Animal / 19-PC game / 550-Dinosaur / \n",
"3-Concert / 7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"9-Music video / 1976-Nostale / \n",
"10-Animal / 17-Racing / 80-Horse / 384-Horse racing / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / \n",
"0-Games / 2-Video game / 243-Sonic the Hedgehog / 4595-Sonic & All-Stars Racing Transformed / \n",
"99-Christmas / 444-Family / 466-Gift / \n",
"45-Cosmetics / 282-Skin / \n",
"5-Dance / 66-Bollywood / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 40-Road / 2807-BMW S1000RR / \n",
"0-Games / 2-Video game / 162-Medicine / \n",
"361-Flamenco / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 30-Drummer / 39-Drums / \n",
"0-Games / 90-Sports game / 180-FIFA 15 / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / 257-Counter-Strike / 1109-Counter-Strike Online / \n",
"93-Comedy / \n",
"379-Touhou Project / 2170-The Embodiment of Scarlet Devil / \n",
"13-Musical ensemble / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 947-Muffler / \n",
"1-Vehicle / 4-Car / 47-Personal computer / 70-Driving / 145-Tablet computer / \n",
"28-Fashion / 390-Model / \n",
"6-Animation / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / 265-Electronic keyboard / \n",
"425-City / \n",
"0-Games / 2-Video game / 505-Mega Man / 2474-Mega Man Battle Network / 4482-Mega Man Battle Network 6 / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 277-Sport utility vehicle / 4493-Nissan X-Trail / \n",
"1-Vehicle / 4-Car / 1321-Škoda Auto / 2690-Škoda Octavia / \n",
"33-Weapon / 242-Paper / \n",
"10-Animal / \n",
"0-Games / 8-Football / 90-Sports game / 180-FIFA 15 / \n",
"0-Games / 8-Football / 90-Sports game / 180-FIFA 15 / \n",
"20-Trailer / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 73-Ball / 81-Athlete / \n",
"93-Comedy / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 6-Animation / 2984-.hack / \n",
"1-Vehicle / 50-Aircraft / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 223-Weather / 266-Helicopter / 392-Radio-controlled helicopter / 499-Cloud / \n",
"3-Concert / 30-Drummer / \n",
"3-Concert / 16-Performance art / \n",
"150-Violin / \n",
"66-Bollywood / \n",
"205-Newscaster / 223-Weather / \n",
"1-Vehicle / 8-Football / 11-Motorsport / 17-Racing / 172-Motocross / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 92-Radio-controlled model / 129-Aviation / 149-Model aircraft / 160-Radio-controlled aircraft / 266-Helicopter / 392-Radio-controlled helicopter / \n",
"7-Musician / 14-Guitar / 24-String instrument / 376-Flute / 726-Clarinet / \n",
"0-Games / 6-Animation / 15-Cartoon / 97-Drawing / 2459-Ceiling fan / \n",
"3-Concert / 412-Monster / \n",
"9-Music video / \n",
"219-Accordion / 655-Diatonic button accordion / \n",
"6-Animation / 10-Animal / 132-Bird / \n",
"10-Animal / 132-Bird / 217-Hunting / \n",
"0-Games / 2-Video game / 637-Five Nights at Freddy's / 1193-Orange / 2500-Pear / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 135-Drifting / 447-Sand / 935-Off-road racing / 975-Dune / \n",
"359-Harry Potter (Literary Series) / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 651-Stock car racing / \n",
"0-Games / 1628-Cabal Online / 2521-Jupiter / 3790-Granado Espada / \n",
"10-Animal / 80-Horse / \n",
"0-Games / 159-World of Warcraft / 171-Warcraft / 1213-World of Warcraft: Wrath of the Lich King / \n",
"10-Animal / 22-Nature / 42-Fishing / 147-River / 1311-Amazon.com; Inc. / \n",
"0-Games / 6-Animation / 15-Cartoon / \n",
"42-Fishing / 573-Paragliding / \n",
"46-Choir / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"211-RuneScape / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / \n",
"14-Guitar / \n",
"6-Animation / 15-Cartoon / 125-Television / \n",
"6-Animation / 15-Cartoon / 61-Art / 97-Drawing / 183-Manga / 396-One Piece / 408-Sketch / 528-Pencil / \n",
"1-Vehicle / 4-Car / 18-Outdoor recreation / 40-Road / 167-Four-wheel drive / 197-Trail / 198-Off-road vehicle / 277-Sport utility vehicle / 327-Mud / 332-Mercedes-Benz / 3279-Mercedes-Benz G-Class / \n",
"3-Concert / 325-Vampire / \n",
"1-Vehicle / 50-Aircraft / 151-Landing / 266-Helicopter / \n",
"628-Portrait / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / \n",
"28-Fashion / 45-Cosmetics / 56-Hair / \n",
"0-Games / 2-Video game / 910-Dragon Quest / 4411-Dragon Quest V: Hand of the Heavenly Bride / \n",
"10-Animal / 80-Horse / \n",
"12-Food / 26-Cooking / 32-Recipe / 109-Cooking show / 226-Dessert / 437-Cream / 454-Sauce / 475-Cheese / 2964-Cream cheese / \n",
"9-Music video / 20-Trailer / 61-Art / 479-Graffiti / \n",
"1-Vehicle / 4-Car / \n",
"83-Skateboarding / 108-Skateboard / \n",
"1-Vehicle / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 220-Mountain bike / 284-Mountain biking / 332-Mercedes-Benz / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 651-Stock car racing / 728-Dirt track racing / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 81-Athlete / \n",
"0-Games / 2-Video game / 111-PlayStation 3 / 237-Final Fantasy / 1131-Final Fantasy XIII / 2506-Final Fantasy XIII-2 / \n",
"1-Vehicle / 4-Car / 1455-Cadillac / 3681-Cadillac CTS / \n",
"429-Website / \n",
"25-Toy / 140-LEGO / 477-Transformers / \n",
"1-Vehicle / 42-Fishing / 87-Boat / 3108-Fishing vessel / \n",
"1-Vehicle / 4-Car / 88-Machine / 114-Engine / 515-Dodge / \n",
"0-Games / 35-Minecraft / \n",
"9-Music video / 99-Christmas / \n",
"0-Games / 326-Metin2 / \n",
"2716-The Hunger Games / \n",
"31-Disc jockey / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / \n",
"0-Games / 8-Football / 65-Kick / 76-Ball / \n",
"0-Games / 21-Stadium / 59-Winter sport / 77-Arena / 112-Ice skating / 142-Hockey / 262-Ice / 312-Ice rink / \n",
"3-Concert / 5-Dance / \n",
"545-Quartet (ensemble) / \n",
"970-Monkey / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 96-Soldier / 138-Call of Duty: Black Ops II / 2434-Call of Duty / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / \n",
"20-Trailer / \n",
"5-Dance / \n",
"0-Games / 3-Concert / 724-Rock Band / \n",
"5-Dance / \n",
"3-Concert / 5-Dance / 16-Performance art / 66-Bollywood / \n",
"3-Concert / 68-Lighting / \n",
"3-Concert / 7-Musician / 16-Performance art / 30-Drummer / \n",
"794-Bridge / \n",
"0-Games / 2-Video game / 241-Fighting game / 311-Street Fighter / 4688-Skullgirls / \n",
"8-Football / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / 30-Drummer / 39-Drums / 44-Drums / \n",
"0-Games / 1737-Dance Dance Revolution / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 36-Piano / 38-Orchestra / 376-Flute / 545-Quartet (ensemble) / \n",
"31-Disc jockey / \n",
"3-Concert / 7-Musician / 30-Drummer / 98-Festival / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 55-Video game console / 518-Mario Kart / \n",
"12-Food / 26-Cooking / 96-Soldier / 301-Meal / 389-Restaurant / 1107-United States Navy / 1572-Husband / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / \n",
"1-Vehicle / 4-Car / 114-Engine / 1321-Škoda Auto / \n",
"1146-Wing Chun / \n",
"8-Football / \n",
"3-Concert / 68-Lighting / \n",
"1-Vehicle / 4-Car / 70-Driving / 332-Mercedes-Benz / \n",
"8-Football / \n",
"0-Games / 8-Football / \n",
"5-Dance / 98-Festival / 250-Parade / 253-Carnival / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 30-Drummer / 39-Drums / 44-Drums / \n",
"12-Food / 1943-Bakery / \n",
"20-Trailer / \n",
"6-Animation / 15-Cartoon / 97-Drawing / 475-Cheese / \n",
"10-Animal / 12-Food / 540-Insect / 775-Bee / 1121-Beehive / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"5-Dance / \n",
"69-Bicycle / 1318-Flashlight / \n",
"0-Games / 6-Animation / \n",
"0-Games / 2-Video game / 1447-Skylanders / 2688-Skylanders: Giants / \n",
"1059-Movie theater / 1184-Flag / \n",
"257-Counter-Strike / \n",
"28-Fashion / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 33-Weapon / 34-Action-adventure game / 118-Grand Theft Auto V / 297-Grand Theft Auto IV / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"45-Cosmetics / 56-Hair / \n",
"0-Games / 1-Vehicle / 4-Car / 62-Train / 837-Roblox / \n",
"0-Games / 2-Video game / 6-Animation / 19-PC game / 240-Arcade game / 241-Fighting game / 311-Street Fighter / 461-Street Fighter IV / 563-Ryu / 1331-Ken Masters / 2239-Street Fighter II: The World Warrior / \n",
"0-Games / 2-Video game / 33-Weapon / \n",
"320-Super Smash Bros. / \n",
"121-Photography / 155-Camera / 2487-Canon PowerShot / \n",
"49-School / 342-Windows Media Video / \n",
"45-Cosmetics / 238-Nail / 300-Nail art / 304-Nail polish / 307-Hand / 324-Finger / 338-Manicure / 823-Artificial nails / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 398-The Legend of Zelda / 692-Link / 1228-The Legend of Zelda: Ocarina of Time / \n",
"750-Sailor Moon / 1342-Sailor Moon / \n",
"1-Vehicle / 4-Car / 5-Dance / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"5-Dance / 78-Wedding / 153-Dress / 190-Bride / 3693-Groomsman / \n",
"3-Concert / 16-Performance art / 68-Lighting / 1431-Madison Square Garden / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"425-City / 2128-Pollution / \n",
"1-Vehicle / 87-Boat / \n",
"102-Building / 113-House / 170-Home improvement / 179-Kitchen / 276-Room / 353-Wall / 492-Floor / 879-Ceiling / 1272-Cabinetry / \n",
"359-Harry Potter (Literary Series) / \n",
"0-Games / 2-Video game / 264-Star Wars / 1050-Star Wars: The Old Republic / \n",
"0-Games / 2-Video game / 19-PC game / 686-Diablo III / 872-Diablo / \n",
"296-Saxophone / \n",
"36-Piano / 53-Keyboard / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / \n",
"1-Vehicle / 92-Radio-controlled model / 94-Dashcam / 149-Model aircraft / 160-Radio-controlled aircraft / 223-Weather / 502-Sky / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 126-The Walt Disney Company / 2453-Disney Infinity / \n",
"130-Gymnastics / \n",
"6-Animation / \n",
"5-Dance / 16-Performance art / 66-Bollywood / \n",
"0-Games / 2-Video game / \n",
"6-Animation / 15-Cartoon / 412-Monster / 503-Monster High / \n",
"9-Music video / \n",
"18-Outdoor recreation / 137-Tree / 213-Forest / 673-Saw / 1073-Chainsaw / 1294-Logging / \n",
"3-Concert / \n",
"1-Vehicle / 601-Shark / \n",
"10-Animal / 80-Horse / 247-Stallion / 404-Livestock / \n",
"1-Vehicle / 4-Car / 158-Tire / 277-Sport utility vehicle / 360-Jeep / \n",
"33-Weapon / \n",
"56-Hair / 340-Afro-textured hair / 733-Washing / \n",
"28-Fashion / \n",
"1-Vehicle / 4-Car / 154-Wheel / 158-Tire / 493-Rim / \n",
"5-Dance / 762-Angel / \n",
"10-Animal / 71-Dog / \n",
"0-Games / 54-Highlight film / 256-Tennis / \n",
"0-Games / 2-Video game / 55-Video game console / \n",
"6-Animation / 15-Cartoon / 126-The Walt Disney Company / 2696-Cinderella / 3092-Cinderella / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"95-Talent show / 115-Ballet / 130-Gymnastics / 1255-Ribbon / \n",
"350-Circus / 788-Juggling / 882-Pen / \n",
"916-Spa / 3386-Hot tub / \n",
"5-Dance / 10-Animal / 31-Disc jockey / 80-Horse / \n",
"12-Food / 49-School / 1152-Plush / 1707-Microwave oven / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 2-Video game / 263-Light / \n",
"6-Animation / 15-Cartoon / \n",
"31-Disc jockey / \n",
"3-Concert / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 81-Athlete / \n",
"1-Vehicle / 4-Car / 449-Honda / 2900-Honda Fit / \n",
"12-Food / 26-Cooking / 52-Dish / 58-Cuisine / \n",
"248-Hotel / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / \n",
"1-Vehicle / 4-Car / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / 242-Paper / 3935-Papier-mâché / \n",
"1-Vehicle / 12-Food / 18-Outdoor recreation / 74-Truck / 617-Maize / \n",
"12-Food / 52-Dish / 58-Cuisine / 215-Eating / 771-Noodle / 1539-Ramen / 3459-Instant noodle / \n",
"6-Animation / \n",
"3-Concert / 7-Musician / \n",
"83-Skateboarding / 108-Skateboard / \n",
"56-Hair / 106-Hairstyle / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 230-Brass instrument / \n",
"5-Dance / 9-Music video / \n",
"12-Food / 348-Fruit / 2995-Blueberry / 3020-Raspberry / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 85-Combat / 96-Soldier / 194-Call of Duty: Modern Warfare 2 / 216-Call of Duty: Modern Warfare 3 / 302-Call of Duty 4: Modern Warfare / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 378-Drag racing / \n",
"0-Games / 6-Animation / 15-Cartoon / 2648-Larva / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 42-Fishing / 147-River / 268-Recreational fishing / 2094-Rainbow trout / \n",
"12-Food / 26-Cooking / 32-Recipe / 290-Pony / 829-Bead / 859-Beadwork / \n",
"31-Disc jockey / \n",
"0-Games / 2-Video game / 803-DC Comics / 1311-Amazon.com; Inc. / 1515-DC Universe Online / \n",
"6-Animation / \n",
"25-Toy / 126-The Walt Disney Company / 354-Egg / 417-Play-Doh / 611-Kinder Surprise / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"10-Animal / 2735-Raccoon / \n",
"93-Comedy / 355-Comedian / \n",
"0-Games / 3235-Monopoly / \n",
"13-Musical ensemble / \n",
"6-Animation / 15-Cartoon / 2644-Fantage / \n",
"10-Animal / 48-Pet / 71-Dog / 1839-Rottweiler / 2049-Police dog / \n",
"20-Trailer / \n",
"1-Vehicle / 4-Car / 11-Motorsport / \n",
"256-Tennis / \n",
"5-Dance / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / \n",
"205-Newscaster / \n",
"35-Minecraft / \n",
"49-School / 835-Backpack / 2454-Notebook / \n",
"0-Games / 2-Video game / 55-Video game console / 176-PlayStation 4 / 314-Xbox One / 386-Xbox / \n",
"5-Dance / \n",
"3-Concert / 7-Musician / \n",
"1-Vehicle / 4-Car / 517-Audi / 2420-Audi TT / \n",
"9-Music video / \n",
"2-Video game / \n",
"5-Dance / 21-Stadium / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 92-Radio-controlled model / 202-Radio-controlled car / 935-Off-road racing / 3419-Truggy / \n",
"1-Vehicle / 4-Car / 234-Ford / 2288-Ford Escort / \n",
"1-Vehicle / 17-Racing / 18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 555-Longboard / 667-Longboarding / \n",
"2-Video game / 6-Animation / 20-Trailer / 320-Super Smash Bros. / \n",
"3-Concert / \n",
"25-Toy / 140-LEGO / 264-Star Wars / 852-Lego Star Wars / 3429-Yoda / \n",
"46-Choir / \n",
"0-Games / 781-Pachinko / \n",
"0-Games / 2-Video game / 638-Gears of War / 773-Gears of War / 1425-Gears of War 2 / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 178-Samsung Galaxy / 1379-HTC One / 2328-HTC One (M8) / \n",
"812-Plastic / 1008-Kite / \n",
"1-Vehicle / 4-Car / \n",
"122-Weight training / \n",
"122-Weight training / \n",
"5-Dance / 16-Performance art / \n",
"1-Vehicle / 4-Car / 1148-Hot rod / \n",
"0-Games / 54-Highlight film / 60-Basketball / 76-Ball / 3291-Basketball Coach / \n",
"31-Disc jockey / \n",
"3-Concert / \n",
"3-Concert / 5-Dance / \n",
"12-Food / 1489-Coffeemaker / \n",
"593-Moon / 1181-Mars / \n",
"1-Vehicle / 11-Motorsport / 20-Trailer / 27-Motorcycle / 105-Motorcycling / 172-Motocross / \n",
"6-Animation / 434-Ninja / \n",
"0-Games / 1029-Aion: The Tower of Eternity / \n",
"56-Hair / 106-Hairstyle / 405-Braid / \n",
"12-Food / 113-House / 3150-Supper / \n",
"31-Disc jockey / \n",
"18-Outdoor recreation / 59-Winter sport / 112-Ice skating / 262-Ice / 316-Figure skating / 833-Ice skate / \n",
"2-Video game / 243-Sonic the Hedgehog / 2028-Sonic World Adventure / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 194-Call of Duty: Modern Warfare 2 / \n",
"0-Games / 51-Strategy video game / 643-Hero / 1687-Heroes of Newerth / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 70-Driving / 245-BMW / 344-Coupé / 3549-BMW M6 / \n",
"6-Animation / 15-Cartoon / 61-Art / 89-Comics / 123-Naruto / 183-Manga / 285-Sasuke Uchiha / 1556-Dōjinshi / \n",
"53-Keyboard / 107-Musical keyboard / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 47-Personal computer / 297-Grand Theft Auto IV / \n",
"0-Games / 88-Machine / 535-Slot machine / 3599-The Wonderful Wizard of Oz / \n",
"18-Outdoor recreation / 42-Fishing / 548-Parachuting / \n",
"0-Games / 19-PC game / 332-Mercedes-Benz / 411-MapleStory / \n",
"3092-Cinderella / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / 44-Drums / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 231-Supercar / 270-Sedan / 378-Drag racing / 1122-Mitsubishi Lancer Evolution / \n",
"1-Vehicle / 103-Rail transport / \n",
"372-Playing card / 1191-Tarot / \n",
"9-Music video / \n",
"376-Flute / 4605-Bansuri / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / \n",
"5-Dance / 16-Performance art / \n",
"1-Vehicle / 4-Car / 270-Sedan / 344-Coupé / 1466-Pontiac / 1493-Supercharger / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"0-Games / 2-Video game / 20-Trailer / 35-Minecraft / 843-The Doctor / \n",
"42-Fishing / \n",
"3-Concert / \n",
"0-Games / 211-RuneScape / 1035-Bank / \n",
"25-Toy / 89-Comics / 435-Action figure / 2121-Marvel Legends / \n",
"93-Comedy / 341-Sketch comedy / 355-Comedian / 416-Apartment / \n",
"117-Boxing / 399-Kickboxing / \n",
"237-Final Fantasy / 730-Final Fantasy VII / \n",
"7-Musician / 14-Guitar / 100-Electric guitar / 606-String / 4002-Eight-string guitar / \n",
"75-Wrestling / \n",
"61-Art / 148-Painting / 303-Paint / 3928-Still life / \n",
"0-Games / 60-Basketball / 168-Basketball moves / 255-Slam dunk / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / \n",
"1-Vehicle / 10-Animal / 48-Pet / 71-Dog / 87-Boat / \n",
"0-Games / 3-Concert / 60-Basketball / \n",
"0-Games / 2-Video game / 779-Point Blank / \n",
"10-Animal / 80-Horse / 438-Mare / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 297-Grand Theft Auto IV / \n",
"1-Vehicle / 262-Ice / \n",
"36-Piano / 53-Keyboard / \n",
"59-Winter sport / 112-Ice skating / 316-Figure skating / \n",
"36-Piano / \n",
"0-Games / 411-MapleStory / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / \n",
"0-Games / 2-Video game / 55-Video game console / 199-Wii / 243-Sonic the Hedgehog / 292-Handheld game console / 335-Nintendo 3DS / 3016-Mario & Sonic at the Olympic Games / \n",
"0-Games / 8-Football / \n",
"1-Vehicle / 124-Tractor / \n",
"0-Games / 2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / 178-Samsung Galaxy / 3361-Samsung Galaxy Y / \n",
"5-Dance / 66-Bollywood / 78-Wedding / \n",
"5-Dance / 13-Musical ensemble / 44-Drums / 253-Carnival / \n",
"120-Winter / 253-Carnival / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 74-Truck / 84-Snow / 120-Winter / 1145-Winter storm / \n",
"0-Games / 2-Video game / 19-PC game / 34-Action-adventure game / 2205-Bully / \n",
"230-Brass instrument / 296-Saxophone / 1006-Tenor saxophone / \n",
"1-Vehicle / 4-Car / 25-Toy / 155-Camera / 1169-Hot Wheels / \n",
"170-Home improvement / 1314-Garage / \n",
"3-Concert / 5-Dance / 16-Performance art / 133-Nightclub / \n",
"3-Concert / 16-Performance art / \n",
"28-Fashion / 1459-Magazine / \n",
"0-Games / 2-Video game / 55-Video game console / 111-PlayStation 3 / 127-Xbox 360 / 181-Xbox / \n",
"0-Games / 75-Wrestling / 116-Pokémon / 186-Pokémon / 1204-Pokémon Ruby and Sapphire / 1665-Pokémon Emerald / 2236-Emerald / \n",
"3-Concert / 16-Performance art / \n",
"30-Drummer / 39-Drums / 44-Drums / 146-Snare drum / \n",
"36-Piano / \n",
"31-Disc jockey / \n",
"42-Fishing / 222-Surfing / 815-Surfboard / \n",
"5-Dance / \n",
"2-Video game / 4550-Conker's Bad Fur Day / \n",
"5-Dance / \n",
"287-Human swimming / 2605-Medley swimming / \n",
"6-Animation / \n",
"3-Concert / 5-Dance / 16-Performance art / 95-Talent show / \n",
"0-Games / 35-Minecraft / \n",
"0-Games / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 595-Champion / \n",
"1-Vehicle / 33-Weapon / 271-Tank / 1409-Missile / 1894-Cannon / \n",
"6-Animation / \n",
"3-Concert / 7-Musician / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / 164-Beach / \n",
"6-Animation / 9-Music video / 967-Text / \n",
"2411-Howcast / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 297-Grand Theft Auto IV / 658-Grand Theft Auto: The Lost and Damned / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 241-Fighting game / 511-Mortal Kombat / 3859-Mortal Kombat II / 4684-Kung Lao / \n",
"9-Music video / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 189-iPad / 272-iPod / 331-iPod touch / \n",
"0-Games / 211-RuneScape / \n",
"2-Video game / 20-Trailer / 34-Action-adventure game / 380-Assassin's Creed / 1241-Assassin's Creed IV: Black Flag / \n",
"860-Christian Church / \n",
"6-Animation / 15-Cartoon / 396-One Piece / \n",
"10-Animal / 18-Outdoor recreation / 42-Fishing / 91-Fish / \n",
"36-Piano / 53-Keyboard / \n",
"5-Dance / 20-Trailer / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 207-Exhaust system / 344-Coupé / 349-Chevrolet / 482-Asphalt / 952-Chevrolet Corvette / 1040-Street racing / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 928-Borderlands 2 / 2613-Borderlands: The Pre-Sequel! / \n",
"3-Concert / 7-Musician / 14-Guitar / 39-Drums / \n",
"6-Animation / 25-Toy / 89-Comics / 396-One Piece / 2525-Gashapon / \n",
"0-Games / 60-Basketball / 77-Arena / 168-Basketball moves / \n",
"653-Thread / 1082-Knot / 2507-Macramé / 3063-Friendship bracelet / \n",
"1-Vehicle / 59-Winter sport / 84-Snow / 120-Winter / 610-Snowmobile / 1201-Ski-Doo / \n",
"7-Musician / 107-Musical keyboard / 219-Accordion / \n",
"28-Fashion / 120-Winter / 757-Boot / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 81-Athlete / \n",
"6-Animation / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 1983-Infestation: Survivor Stories / \n",
"0-Games / 2-Video game / 1118-Kirby / 1133-Kirby / 4449-King Dedede / \n",
"3-Concert / \n",
"3-Concert / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 508-iPhone 5 / 2038-Credit card / \n",
"0-Games / 1-Vehicle / 2-Video game / 6-Animation / 50-Aircraft / 82-Airplane / 129-Aviation / 451-Cockpit / \n",
"6-Animation / 15-Cartoon / 4122-Marceline the Vampire Queen / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"1465-FOX / \n",
"121-Photography / 155-Camera / \n",
"0-Games / 2-Video game / 2774-Tom Clancy's Rainbow Six / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 109-Cooking show / 179-Kitchen / 597-Soup / 742-Countertop / 1108-Mushroom / 2309-Low-carbohydrate diet / \n",
"28-Fashion / 60-Basketball / 206-Shoe / 333-Nike; Inc. / 368-Sneakers / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 1125-God of War / 1743-God of War / \n",
"9-Music video / 14-Guitar / 24-String instrument / \n",
"102-Building / 162-Medicine / 209-University / \n",
"5-Dance / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / 154-Wheel / 158-Tire / 207-Exhaust system / 806-Chopper / \n",
"3-Concert / 7-Musician / 978-Love song / \n",
"5-Dance / \n",
"5-Dance / 16-Performance art / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 603-StarCraft II: Wings of Liberty / 732-StarCraft / 2353-Heroes of the Storm / 2647-StarCraft II: Heart of the Swarm / \n",
"7-Musician / 13-Musical ensemble / 24-String instrument / 38-Orchestra / 150-Violin / 450-Viola / 639-Fiddle / \n",
"0-Games / 2-Video game / 6-Animation / 701-Beyblade / \n",
"0-Games / 51-Strategy video game / 382-Dota 2 / 549-Defense of the Ancients / \n",
"0-Games / 116-Pokémon / 186-Pokémon / 4521-Deoxys / \n",
"0-Games / 2-Video game / 33-Weapon / \n",
"6-Animation / 97-Drawing / 538-Illustration / 2945-Fan art / 3865-Blue Dragon / \n",
"31-Disc jockey / \n",
"5-Dance / 9-Music video / \n",
"2-Video game / \n",
"12-Food / 1346-Wood carving / \n",
"12-Food / 26-Cooking / 32-Recipe / 442-Bread / \n",
"20-Trailer / \n",
"0-Games / 8-Football / 65-Kick / 1247-Penalty kick / \n",
"5-Dance / 9-Music video / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 194-Call of Duty: Modern Warfare 2 / \n",
"359-Harry Potter (Literary Series) / 4489-Harry Potter and the Philosopher's Stone / \n",
"67-Cycling / 69-Bicycle / 88-Machine / 1067-Lock / 2516-Lock picking / \n",
"61-Art / 148-Painting / \n",
"6-Animation / 2984-.hack / \n",
"46-Choir / \n",
"0-Games / 2-Video game / 20-Trailer / 111-PlayStation 3 / \n",
"0-Games / 35-Minecraft / 55-Video game console / 127-Xbox 360 / 181-Xbox / \n",
"6-Animation / \n",
"0-Games / 2-Video game / 6-Animation / 514-Tekken / 797-Tekken / 1273-Tekken 6 / \n",
"7-Musician / \n",
"31-Disc jockey / \n",
"0-Games / 54-Highlight film / 163-Running / \n",
"42-Fishing / 275-Diving / 381-Underwater diving / 407-Scuba diving / \n",
"6-Animation / 15-Cartoon / 89-Comics / \n",
"6-Animation / \n",
"273-News program / 3257-Hindu temple / \n",
"0-Games / 8-Football / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / 265-Electronic keyboard / \n",
"6-Animation / 25-Toy / \n",
"0-Games / 2-Video game / 1137-Crash Bandicoot / \n",
"9-Music video / \n",
"364-Sitcom / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 247-Stallion / 289-Dressage / 404-Livestock / 438-Mare / \n",
"1-Vehicle / 4-Car / \n",
"66-Bollywood / \n",
"3-Concert / 13-Musical ensemble / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 47-Personal computer / 96-Soldier / 302-Call of Duty 4: Modern Warfare / \n",
"10-Animal / 22-Nature / 537-Zoo / \n",
"28-Fashion / 56-Hair / 106-Hairstyle / \n",
"1271-Beauty salon / \n",
"0-Games / 59-Winter sport / 81-Athlete / 112-Ice skating / 142-Hockey / 262-Ice / 312-Ice rink / \n",
"609-Tattoo / 863-Body piercing / \n",
"1-Vehicle / 4-Car / 84-Snow / 277-Sport utility vehicle / \n",
"59-Winter sport / 281-Snowboarding / 4321-Mountainboarding / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 2857-Yamaha Raptor 700R / \n",
"3-Concert / 7-Musician / 1666-Hammond organ / \n",
"122-Weight training / 204-Gym / 214-Muscle / 469-Bag / 529-Squat / 532-Barbell / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 207-Exhaust system / 231-Supercar / 344-Coupé / 517-Audi / 1537-Audi R8 / 2949-V10 engine / \n",
"6-Animation / 15-Cartoon / 412-Monster / 503-Monster High / \n",
"676-Handbag / \n",
"8-Football / 21-Stadium / \n",
"25-Toy / 140-LEGO / 264-Star Wars / 852-Lego Star Wars / 2576-Clone trooper / \n",
"8-Football / 81-Athlete / \n",
"31-Disc jockey / \n",
"12-Food / 1870-Chewing gum / 1943-Bakery / \n",
"1-Vehicle / 4-Car / 74-Truck / 92-Radio-controlled model / 167-Four-wheel drive / 202-Radio-controlled car / \n",
"23-Mobile phone / 29-Smartphone / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 651-Stock car racing / 728-Dirt track racing / \n",
"0-Games / 1467-Pump It Up / 2651-Pump It Up Fiesta / 3591-Pump It Up Fiesta 2 / \n",
"0-Games / 2-Video game / 19-PC game / 928-Borderlands 2 / \n",
"93-Comedy / \n",
"0-Games / 2-Video game / 127-Xbox 360 / 445-Guitar Hero / 965-Guitar Hero III: Legends of Rock / \n",
"31-Disc jockey / \n",
"8-Football / 21-Stadium / \n",
"3-Concert / 16-Performance art / 93-Comedy / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"46-Choir / \n",
"6-Animation / 2435-Tokyo Ghoul / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"12-Food / 32-Recipe / 617-Maize / 684-Ice cream / \n",
"5-Dance / \n",
"455-Unidentified flying object / 2918-Aurora / \n",
"1-Vehicle / 189-iPad / 4558-Spektrum RC / \n",
"6-Animation / 15-Cartoon / 61-Art / 97-Drawing / 408-Sketch / 528-Pencil / \n",
"9-Music video / \n",
"49-School / \n",
"14-Guitar / 63-Acoustic guitar / 361-Flamenco / 2380-Flamenco guitar / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 361-Flamenco / \n",
"102-Building / 113-House / 521-Metal / 2428-Barn / \n",
"102-Building / 113-House / 170-Home improvement / 276-Room / 353-Wall / 1350-Plaster / \n",
"86-Plant / 137-Tree / 1208-Coconut / \n",
"163-Running / 674-Marathon / \n",
"68-Lighting / 263-Light / \n",
"0-Games / 1-Vehicle / 2-Video game / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 172-Motocross / 2930-MX vs. ATV / \n",
"23-Mobile phone / 29-Smartphone / \n",
"6-Animation / 15-Cartoon / 20-Trailer / 123-Naruto / 285-Sasuke Uchiha / 3232-Naruto Uzumaki / \n",
"0-Games / 2-Video game / 584-Super Mario Bros. / 1170-Super Mario World / \n",
"6-Animation / 15-Cartoon / 396-One Piece / \n",
"5-Dance / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 454-Sauce / 620-Barbecue / 966-Sandwich / 1517-Spaghetti / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 760-iPhone 5s / \n",
"56-Hair / 106-Hairstyle / \n",
"0-Games / 2-Video game / 19-PC game / 638-Gears of War / 773-Gears of War / \n",
"313-Book / \n",
"55-Video game console / 581-PlayStation / \n",
"56-Hair / 106-Hairstyle / 1359-Hair iron / \n",
"85-Combat / 117-Boxing / 399-Kickboxing / \n",
"6-Animation / 28-Fashion / \n",
"6-Animation / 126-The Walt Disney Company / \n",
"2-Video game / 47-Personal computer / 131-Computer / \n",
"0-Games / 489-Board game / 922-Tabletop game / \n",
"3343-The Real Housewives / \n",
"1-Vehicle / 4-Car / 74-Truck / 158-Tire / 167-Four-wheel drive / 375-Pickup truck / 3541-Nissan Navara / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 194-Call of Duty: Modern Warfare 2 / 302-Call of Duty 4: Modern Warfare / \n",
"89-Comics / 1791-Stargate / \n",
"3770-Toothpaste / \n",
"47-Personal computer / 831-Plants vs. Zombies / \n",
"756-Tibia / 1356-Paladin / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / 30-Drummer / 39-Drums / 44-Drums / \n",
"0-Games / 467-Chipmunk / 3104-Tree squirrel / \n",
"25-Toy / 83-Skateboarding / 1105-Fingerboard / \n",
"1-Vehicle / 4-Car / 299-Volkswagen Passenger Cars / 526-Volkswagen / 1761-Volkswagen Jetta / \n",
"347-Climbing / \n",
"8-Football / 73-Ball / \n",
"102-Building / 113-House / 310-The Sims / 328-Simulation video game / 564-The Sims 3 / \n",
"3-Concert / 16-Performance art / 30-Drummer / 68-Lighting / \n",
"1-Vehicle / 18-Outdoor recreation / 42-Fishing / 50-Aircraft / 548-Parachuting / 573-Paragliding / 765-Parachute / 778-Gliding / 1561-Paramotor / 1602-Hang gliding / \n",
"1-Vehicle / \n",
"1-Vehicle / 4-Car / 270-Sedan / 332-Mercedes-Benz / 891-Mercedes-AMG / 1310-Mercedes-Benz C-Class / 2317-Twin-turbo / 2365-Mercedes-Benz S-Class / \n",
"9-Music video / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 1091-Saints Row / 2470-Hitman: Absolution / 2495-Saints Row 2 / \n",
"1109-Counter-Strike Online / 3712-Labyrinth / \n",
"10-Animal / 17-Racing / 80-Horse / 384-Horse racing / \n",
"14-Guitar / 56-Hair / 106-Hairstyle / \n",
"402-Jewellery / 859-Beadwork / 907-Necklace / \n",
"25-Toy / 749-Figurine / \n",
"78-Wedding / 190-Bride / \n",
"0-Games / 8-Football / 54-Highlight film / 76-Ball / 767-Goalkeeper / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 158-Tire / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 573-Paragliding / 1561-Paramotor / \n",
"452-Watch / 2168-G-Shock / \n",
"0-Games / 59-Winter sport / 81-Athlete / 112-Ice skating / 142-Hockey / 312-Ice rink / \n",
"0-Games / 19-PC game / 51-Strategy video game / 165-League of Legends / \n",
"5-Dance / 16-Performance art / \n",
"3-Concert / \n",
"20-Trailer / \n",
"0-Games / 2-Video game / 33-Weapon / 1382-Half-Life 2 / \n",
"12-Food / 56-Hair / 106-Hairstyle / 340-Afro-textured hair / 857-Afro / \n",
"3-Concert / 5-Dance / \n",
"22-Nature / 86-Plant / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 199-Wii / 1447-Skylanders / 2321-Skylanders: Spyro's Adventure / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 3046-Maltese / \n",
"61-Art / 82-Airplane / 242-Paper / 370-Origami / 1933-Paper plane / \n",
"3-Concert / 5-Dance / \n",
"9-Music video / \n",
"110-Album / \n",
"45-Cosmetics / 235-Lipstick / 291-Rouge / 1012-Lip gloss / \n",
"122-Weight training / 214-Muscle / 529-Squat / \n",
"6-Animation / 15-Cartoon / 1599-Bone / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 19-PC game / 34-Action-adventure game / 111-PlayStation 3 / 118-Grand Theft Auto V / \n",
"31-Disc jockey / \n",
"6-Animation / 10-Animal / 15-Cartoon / 3175-Hide-and-seek / \n",
"1-Vehicle / 4-Car / 41-Sports car / 650-Convertible / \n",
"14-Guitar / \n",
"6-Animation / 15-Cartoon / 4410-PriPara / \n",
"0-Games / 2-Video game / 19-PC game / 310-The Sims / 328-Simulation video game / 1713-The Sims / \n",
"42-Fishing / 164-Beach / 213-Forest / \n",
"260-Prayer / \n",
"56-Hair / 106-Hairstyle / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / 88-Machine / 608-Wheelie / \n",
"31-Disc jockey / \n",
"28-Fashion / 45-Cosmetics / 56-Hair / 106-Hairstyle / 583-Wig / 607-Lace / \n",
"0-Games / 756-Tibia / \n",
"3-Concert / 5-Dance / \n",
"0-Games / 60-Basketball / 76-Ball / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 70-Driving / 212-Highway / 2892-GSX-R750 / \n",
"390-Model / \n",
"83-Skateboarding / 108-Skateboard / 482-Asphalt / 555-Longboard / 556-Skateboarding trick / 667-Longboarding / 1315-Kickflip / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 19-PC game / 34-Action-adventure game / 118-Grand Theft Auto V / 1035-Bank / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 188-Airline / 1340-Airbus A380 / \n",
"0-Games / 2-Video game / 33-Weapon / 229-Halo / 470-Halo 3 / 644-Halo: Reach / \n",
"0-Games / 4554-Marvel Super Hero Squad Online / \n",
"10-Animal / 22-Nature / 86-Plant / 540-Insect / 642-Leaf / 775-Bee / 784-Honey / 1121-Beehive / \n",
"6-Animation / 1054-Vacuum cleaner / \n",
"56-Hair / 106-Hairstyle / 583-Wig / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 143-Railroad car / 428-Rail transport modelling / 1450-Model building / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 18-Outdoor recreation / 27-Motorcycle / 105-Motorcycling / 172-Motocross / 491-Enduro / \n",
"6-Animation / \n",
"61-Art / 148-Painting / \n",
"247-Stallion / 438-Mare / 1871-Foal / 2018-Arabian horse / \n",
"5-Dance / \n",
"9-Music video / \n",
"3-Concert / 5-Dance / 7-Musician / 13-Musical ensemble / 38-Orchestra / 130-Gymnastics / 1835-Majorette / \n",
"1-Vehicle / 4-Car / 114-Engine / 517-Audi / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"66-Bollywood / \n",
"3-Concert / 68-Lighting / \n",
"8-Football / 21-Stadium / \n",
"0-Games / 2-Video game / 116-Pokémon / \n",
"113-House / 1731-Haunted house / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 19-PC game / 506-Garry's Mod / 2290-Trouble in Terrorist Town / 3841-The Hidden / \n",
"0-Games / 2-Video game / 127-Xbox 360 / \n",
"5-Dance / 889-Swing / \n",
"5-Dance / 219-Accordion / \n",
"0-Games / 756-Tibia / \n",
"101-Telephone / 131-Computer / \n",
"9-Music video / \n",
"20-Trailer / \n",
"75-Wrestling / 85-Combat / \n",
"0-Games / 2-Video game / 10-Animal / 80-Horse / 384-Horse racing / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 89-Comics / 396-One Piece / \n",
"2-Video game / 20-Trailer / 89-Comics / 294-Batman / 838-Batman: Arkham / 2082-Catwoman / 2813-Batmobile / 3473-Nightwing / \n",
"12-Food / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 378-Drag racing / 449-Honda / 552-Honda Civic / \n",
"1-Vehicle / 4-Car / 17-Racing / 92-Radio-controlled model / 202-Radio-controlled car / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 413-PlayStation 2 / 3510-Ridge Racer / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 2-Video game / 6-Animation / 241-Fighting game / 514-Tekken / 797-Tekken / 1273-Tekken 6 / \n",
"25-Toy / 626-Puzzle / 631-Cube / 740-Rubik's Cube / \n",
"18-Outdoor recreation / 22-Nature / 147-River / 213-Forest / \n",
"6-Animation / 3144-Tinker Bell / \n",
"55-Video game console / 127-Xbox 360 / 181-Xbox / 832-Headset / \n",
"110-Album / \n",
"3-Concert / 68-Lighting / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 291-Rouge / \n",
"0-Games / 2-Video game / 20-Trailer / 34-Action-adventure game / 111-PlayStation 3 / 127-Xbox 360 / 656-Metal Gear / 1419-GameSpot / 2698-Metal Gear Rising: Revengeance / \n",
"0-Games / 60-Basketball / 168-Basketball moves / \n",
"9-Music video / 14-Guitar / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / 735-Kamen Rider Series / 1360-Magician / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 138-Call of Duty: Black Ops II / \n",
"9-Music video / \n",
"3-Concert / 44-Drums / \n",
"49-School / \n",
"3-Concert / 16-Performance art / \n",
"3-Concert / \n",
"6-Animation / 15-Cartoon / 259-Fire / 434-Ninja / \n",
"12-Food / 26-Cooking / \n",
"1-Vehicle / 4-Car / \n",
"3-Concert / 7-Musician / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 169-Rallying / \n",
"0-Games / 35-Minecraft / \n",
"45-Cosmetics / \n",
"0-Games / 2-Video game / 34-Action-adventure game / \n",
"0-Games / 30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"0-Games / 6-Animation / 97-Drawing / 1563-Stick figure / \n",
"1-Vehicle / 50-Aircraft / 129-Aviation / 388-Fighter aircraft / 2954-McDonnell Douglas F-15 Eagle / 4141-Lock On: Modern Air Combat / \n",
"5-Dance / \n",
"8-Football / 54-Highlight film / \n",
"5-Dance / 430-Ballroom dance / \n",
"1-Vehicle / 27-Motorcycle / 154-Wheel / 1124-Yamaha YZF-R1 / \n",
"1-Vehicle / 4-Car / 270-Sedan / 332-Mercedes-Benz / 1310-Mercedes-Benz C-Class / 2603-Mercedes-Benz E-Class / \n",
"6-Animation / 25-Toy / 140-LEGO / 434-Ninja / 865-Turtle / 884-Lego minifigure / 951-Teenage Mutant Ninja Turtles / 3667-Raphael / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 254-Runway / 388-Fighter aircraft / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 210-Cookware and bakeware / 274-Meat / 454-Sauce / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 84-Snow / \n",
"123-Naruto / \n",
"20-Trailer / \n",
"3-Concert / 16-Performance art / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 210-Cookware and bakeware / 239-Roasting / 274-Meat / 542-Potato / 1509-Stew / \n",
"1-Vehicle / 4-Car / 349-Chevrolet / \n",
"6-Animation / 2408-Lucky Star / \n",
"28-Fashion / 209-University / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 19-PC game / 928-Borderlands 2 / \n",
"0-Games / 2-Video game / 43-Call of Duty / 196-Eye / 609-Tattoo / \n",
"162-Medicine / 780-Fat / 1162-Dietary supplement / \n",
"3-Concert / 133-Nightclub / \n",
"25-Toy / 264-Star Wars / 435-Action figure / 2576-Clone trooper / \n",
"0-Games / 8-Football / \n",
"96-Soldier / 758-Air force / 2202-Armed forces / \n",
"0-Games / 2-Video game / 33-Weapon / \n",
"8-Football / \n",
"86-Plant / 136-Gardening / 137-Tree / 213-Forest / 279-Garden / \n",
"10-Animal / 48-Pet / 970-Monkey / 1948-Ape / 3157-Chimpanzee / \n",
"3-Concert / 5-Dance / 31-Disc jockey / 133-Nightclub / \n",
"7-Musician / 13-Musical ensemble / 30-Drummer / 44-Drums / 296-Saxophone / \n",
"2-Video game / 739-Guild Wars / \n",
"5-Dance / 16-Performance art / 115-Ballet / \n",
"125-Television / \n",
"93-Comedy / 125-Television / \n",
"1-Vehicle / 4-Car / 34-Action-adventure game / 41-Sports car / 297-Grand Theft Auto IV / \n",
"1-Vehicle / 2-Video game / 864-Speedometer / 1099-Euro Truck Simulator 2 / \n",
"319-Robot / \n",
"20-Trailer / \n",
"2207-Kemenche / \n",
"6-Animation / 35-Minecraft / \n",
"5-Dance / 16-Performance art / \n",
"279-Garden / 585-Lawn / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 291-Rouge / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 86-Plant / 210-Cookware and bakeware / 620-Barbecue / 662-Pizza / 752-Pasta / 1108-Mushroom / 1230-Italian food / 3021-Tomato sauce / \n",
"61-Art / 479-Graffiti / \n",
"0-Games / 2176-Fist of the North Star / \n",
"3-Concert / \n",
"7-Musician / 46-Choir / \n",
"3-Concert / \n",
"1-Vehicle / 259-Fire / 293-Slide show / 371-Firefighter / 796-Fire engine / 1020-Emergency vehicle / 1633-Fire station / \n",
"14-Guitar / 1970-Axe / \n",
"25-Toy / 524-Littlest Pet Shop / 587-Clay / \n",
"10-Animal / 80-Horse / \n",
"20-Trailer / \n",
"162-Medicine / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 157-Jet aircraft / 388-Fighter aircraft / 929-Aerobatics / \n",
"0-Games / 10-Animal / \n",
"0-Games / 2-Video game / 20-Trailer / 165-League of Legends / 2004-Riot Games / \n",
"45-Cosmetics / 193-Eye liner / 235-Lipstick / 2716-The Hunger Games / \n",
"6-Animation / \n",
"1-Vehicle / 27-Motorcycle / 67-Cycling / 69-Bicycle / 114-Engine / 154-Wheel / 520-Bicycle frame / \n",
"1163-Carpet / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 272-iPod / 331-iPod touch / 675-iPhone 4S / \n",
"352-Need for Speed / 1583-Toyota Corolla / 1600-Need for Speed: World / \n",
"23-Mobile phone / 29-Smartphone / 345-Loudspeaker / \n",
"28-Fashion / 78-Wedding / 190-Bride / 590-Wedding dress / \n",
"1-Vehicle / 4-Car / 3762-PSA HDi engine / \n",
"1-Vehicle / 17-Racing / 27-Motorcycle / 67-Cycling / 69-Bicycle / 570-Road bicycle racing / \n",
"8-Football / \n",
"6-Animation / 15-Cartoon / 61-Art / \n",
"0-Games / 55-Video game console / 199-Wii / 2750-Mii / 3226-Chucky / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / 265-Electronic keyboard / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 139-Airport / 151-Landing / 187-Takeoff / 188-Airline / 451-Cockpit / \n",
"5-Dance / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 197-Trail / 213-Forest / 220-Mountain bike / 284-Mountain biking / \n",
"0-Games / 379-Touhou Project / 455-Unidentified flying object / 2170-The Embodiment of Scarlet Devil / \n",
"426-Resident Evil / 3514-Resident Evil: Revelations / \n",
"122-Weight training / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 824-Terrier / 2388-Bull Terrier / 3052-Staffordshire Bull Terrier / \n",
"75-Wrestling / \n",
"45-Cosmetics / 56-Hair / \n",
"45-Cosmetics / 238-Nail / 300-Nail art / 304-Nail polish / 307-Hand / 324-Finger / 338-Manicure / 823-Artificial nails / \n",
"238-Nail / 300-Nail art / 304-Nail polish / \n",
"18-Outdoor recreation / 42-Fishing / 164-Beach / 222-Surfing / 233-Ocean / 672-Bodyboarding / \n",
"0-Games / 73-Ball / 326-Metin2 / \n",
"5-Dance / 16-Performance art / \n",
"170-Home improvement / 488-Door / 1078-Installation art / 1314-Garage / 3457-Garage door / \n",
"31-Disc jockey / 861-Mixing console / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 215-Eating / 301-Meal / 389-Restaurant / 589-Grilling / 1438-Street food / \n",
"1-Vehicle / 4-Car / 62-Train / 286-Bus / \n",
"0-Games / 2-Video game / 35-Minecraft / 243-Sonic the Hedgehog / 2327-Sonic the Hedgehog / \n",
"1-Vehicle / 87-Boat / 344-Coupé / 498-Motorboat / 579-Yacht / \n",
"28-Fashion / 153-Dress / \n",
"117-Boxing / \n",
"0-Games / 2-Video game / 21-Stadium / 35-Minecraft / 77-Arena / 79-American football / 127-Xbox 360 / 181-Xbox / \n",
"9-Music video / \n",
"9-Music video / 541-Black-and-white / \n",
"18-Outdoor recreation / 42-Fishing / \n",
"370-Origami / \n",
"14-Guitar / 100-Electric guitar / \n",
"6-Animation / 1527-Pretty Cure / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 921-Grand Theft Auto: Vice City / \n",
"1-Vehicle / 18-Outdoor recreation / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 266-Helicopter / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 85-Combat / 380-Assassin's Creed / 1140-Assassin's Creed III / 1241-Assassin's Creed IV: Black Flag / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / \n",
"2-Video game / 47-Personal computer / 208-Wood / 1045-Computer case / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 96-Soldier / 257-Counter-Strike / 464-First-person Shooter / \n",
"205-Newscaster / \n",
"0-Games / 8-Football / 21-Stadium / 81-Athlete / \n",
"117-Boxing / \n",
"10-Animal / 48-Pet / 166-Cat / 346-Kitten / 2924-Persian cat / \n",
"0-Games / 2-Video game / 55-Video game console / 292-Handheld game console / 1446-Game Boy / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 85-Combat / 693-Sword / 1166-Dark Souls II / \n",
"0-Games / 8-Football / \n",
"3-Concert / 5-Dance / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / \n",
"6-Animation / 325-Vampire / 4524-Rosario + Vampire / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 79-American football / 81-Athlete / \n",
"904-Cocktail / 1128-Glitter / 2184-Lush / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 219-Accordion / \n",
"45-Cosmetics / 162-Medicine / 4597-Jaw / \n",
"0-Games / 2-Video game / 33-Weapon / 506-Garry's Mod / \n",
"0-Games / 51-Strategy video game / 165-League of Legends / \n",
"12-Food / 3117-Match / \n",
"20-Trailer / \n",
"0-Games / 2-Video game / 194-Call of Duty: Modern Warfare 2 / \n",
"0-Games / 7-Musician / 14-Guitar / 1762-Rock Band 3 / \n",
"0-Games / 2-Video game / 43-Call of Duty / 216-Call of Duty: Modern Warfare 3 / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 70-Driving / 94-Dashcam / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 520-Bicycle frame / 645-Road bicycle / \n",
"36-Piano / 53-Keyboard / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 30-Drummer / 38-Orchestra / 39-Drums / 2116-Hard Rock Cafe / \n",
"0-Games / 60-Basketball / 77-Arena / 168-Basketball moves / 255-Slam dunk / \n",
"0-Games / 60-Basketball / 168-Basketball moves / 255-Slam dunk / \n",
"1353-Bishop / \n",
"1-Vehicle / 50-Aircraft / 129-Aviation / 266-Helicopter / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / \n",
"467-Chipmunk / \n",
"0-Games / 59-Winter sport / 81-Athlete / 112-Ice skating / 142-Hockey / 312-Ice rink / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 64-Transport / 425-City / \n",
"6-Animation / 15-Cartoon / \n",
"5-Dance / 98-Festival / 253-Carnival / \n",
"10-Animal / \n",
"248-Hotel / 916-Spa / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 705-Forza Motorsport / 2106-Forza Horizon 2 / \n",
"1-Vehicle / 50-Aircraft / 451-Cockpit / 2724-North American P-51 Mustang / \n",
"10-Animal / 71-Dog / 385-Face / \n",
"0-Games / 2-Video game / 194-Call of Duty: Modern Warfare 2 / \n",
"1-Vehicle / 27-Motorcycle / \n",
"9-Music video / \n",
"0-Games / 2302-Dying Light / \n",
"0-Games / 75-Wrestling / \n",
"0-Games / 2-Video game / 35-Minecraft / 55-Video game console / 111-PlayStation 3 / 127-Xbox 360 / 176-PlayStation 4 / 181-Xbox / 314-Xbox One / 317-PlayStation / \n",
"425-City / \n",
"355-Comedian / \n",
"0-Games / 257-Counter-Strike / 826-Source Engine / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"45-Cosmetics / 235-Lipstick / \n",
"12-Food / 26-Cooking / 32-Recipe / 179-Kitchen / \n",
"7-Musician / 13-Musical ensemble / 44-Drums / 122-Weight training / 203-Marching band / 820-Bagpipes / 1646-Pipe band / \n",
"12-Food / 26-Cooking / 32-Recipe / 904-Cocktail / 4295-Mojito / \n",
"259-Fire / 1245-Alarm device / \n",
"56-Hair / \n",
"28-Fashion / 1168-Neighbourhood / \n",
"101-Telephone / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / 49-School / \n",
"17-Racing / 41-Sports car / 231-Supercar / 711-Gran Turismo / 953-Gran Turismo 5 / \n",
"1-Vehicle / 4-Car / 403-Toyota / \n",
"6-Animation / 15-Cartoon / 123-Naruto / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"0-Games / 8-Football / 77-Arena / \n",
"0-Games / 6-Animation / 10-Animal / 15-Cartoon / 537-Zoo / 2157-Guppy / \n",
"1-Vehicle / 114-Engine / 854-Diesel engine / \n",
"117-Boxing / 609-Tattoo / 628-Portrait / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / \n",
"93-Comedy / \n",
"9-Music video / \n",
"6-Animation / 15-Cartoon / 126-The Walt Disney Company / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 101-Telephone / 508-iPhone 5 / 760-iPhone 5s / 1572-Husband / \n",
"0-Games / 1-Vehicle / 2-Video game / 27-Motorcycle / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / \n",
"12-Food / \n",
"31-Disc jockey / \n",
"3-Concert / 5-Dance / 16-Performance art / 95-Talent show / \n",
"121-Photography / 541-Black-and-white / \n",
"5-Dance / 13-Musical ensemble / \n",
"9-Music video / \n",
"12-Food / 389-Restaurant / \n",
"23-Mobile phone / 37-Gadget / 72-iPhone / 516-Headphones / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 105-Motorcycling / 1282-Honda CBR series / \n",
"0-Games / 20-Trailer / 60-Basketball / \n",
"1-Vehicle / 18-Outdoor recreation / 136-Gardening / 137-Tree / 2795-Root / \n",
"110-Album / \n",
"2-Video game / 121-Photography / 125-Television / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 164-Beach / 247-Stallion / \n",
"9-Music video / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 118-Grand Theft Auto V / 135-Drifting / \n",
"9-Music video / 153-Dress / \n",
"0-Games / \n",
"40-Road / 67-Cycling / 69-Bicycle / 220-Mountain bike / 520-Bicycle frame / 645-Road bicycle / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"0-Games / 51-Strategy video game / 373-Clash of Clans / \n",
"123-Naruto / 183-Manga / 285-Sasuke Uchiha / 1180-Itachi Uchiha / \n",
"8-Football / \n",
"66-Bollywood / \n",
"3-Concert / 99-Christmas / 3581-Rudolph the Red-Nosed Reindeer / \n",
"2-Video game / 20-Trailer / 229-Halo / 644-Halo: Reach / 2889-343 Industries / 3360-Halo: Combat Evolved Anniversary / \n",
"28-Fashion / \n",
"318-Church / \n",
"8-Football / \n",
"162-Medicine / \n",
"9-Music video / \n",
"0-Games / 60-Basketball / 168-Basketball moves / \n",
"3-Concert / 5-Dance / 13-Musical ensemble / 1253-Ballet Dancer / \n",
"117-Boxing / \n",
"23-Mobile phone / 29-Smartphone / 47-Personal computer / 414-Computer monitor / \n",
"0-Games / 47-Personal computer / 141-Microsoft Windows / \n",
"0-Games / 2-Video game / 19-PC game / 47-Personal computer / \n",
"0-Games / 2-Video game / 311-Street Fighter / 461-Street Fighter IV / 496-Super Street Fighter IV / 1988-Chun-Li / 3981-Fei Long / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"0-Games / 2-Video game / 1777-Ratchet / 1824-Clank / 1859-Ratchet & Clank / \n",
"2-Video game / 20-Trailer / 34-Action-adventure game / 111-PlayStation 3 / 127-Xbox 360 / 181-Xbox / 1028-Devil May Cry / 2344-DmC: Devil May Cry / \n",
"9-Music video / \n",
"8-Football / \n",
"0-Games / 51-Strategy video game / 373-Clash of Clans / \n",
"1-Vehicle / 27-Motorcycle / \n",
"67-Cycling / 69-Bicycle / 221-Skatepark / 377-BMX bike / \n",
"233-Ocean / 246-Furniture / 276-Room / 308-Ship / 605-Cruise ship / 2151-Carnival Cruise Lines / \n",
"1-Vehicle / 20-Trailer / 27-Motorcycle / \n",
"6-Animation / 15-Cartoon / 97-Drawing / 3023-Line Rider / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 261-Tool / 378-Drag racing / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 179-Kitchen / 210-Cookware and bakeware / 3008-Slow cooker / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 509-Classic car / 1148-Hot rod / \n",
"766-President of the United States / 986-Suit / \n",
"3-Concert / 68-Lighting / \n",
"0-Games / 2-Video game / 33-Weapon / 474-Destiny / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 382-Dota 2 / 549-Defense of the Ancients / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 210-Cookware and bakeware / 274-Meat / 589-Grilling / 620-Barbecue / 688-Beef / 841-Butter / 862-Pork / 1085-Barbecue grill / 1115-Steak / \n",
"3-Concert / 5-Dance / 16-Performance art / 46-Choir / \n",
"0-Games / 89-Comics / 387-Spider-Man / \n",
"661-Stretching / \n",
"458-Gold / \n",
"1-Vehicle / 17-Racing / 87-Boat / 579-Yacht / 917-Sail / 1309-Sailing ship / 1957-Luxury yacht / \n",
"8-Football / 595-Champion / \n",
"25-Toy / 140-LEGO / 3181-Lego Legends of Chima / \n",
"31-Disc jockey / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 169-Rallying / 198-Off-road vehicle / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 77-Arena / \n",
"6-Animation / 15-Cartoon / 89-Comics / 385-Face / 1164-Cartoon Network / 1696-Raven / \n",
"49-School / \n",
"0-Games / 8-Football / 85-Combat / \n",
"10-Animal / 48-Pet / 132-Bird / \n",
"0-Games / 2-Video game / 237-Final Fantasy / 1131-Final Fantasy XIII / \n",
"85-Combat / 117-Boxing / 2851-Fight Night / \n",
"0-Games / 2-Video game / 19-PC game / 20-Trailer / 33-Weapon / 43-Call of Duty / 96-Soldier / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / 194-Call of Duty: Modern Warfare 2 / \n",
"1193-Orange / \n",
"1-Vehicle / 87-Boat / 147-River / \n",
"10-Animal / 91-Fish / 126-The Walt Disney Company / 144-Amusement park / 258-Aquarium / 576-Walt Disney World / 716-Coral / 1463-Magic Kingdom / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 456-Go-kart / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 1455-Cadillac / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 352-Need for Speed / 924-Trophy / 1770-Need for Speed: Hot Pursuit / 2545-Need for Speed III: Hot Pursuit / \n",
"0-Games / 2-Video game / 43-Call of Duty / 194-Call of Duty: Modern Warfare 2 / \n",
"1-Vehicle / 4-Car / 74-Truck / 249-Heavy equipment / 1611-Garbage truck / \n",
"1-Vehicle / 12-Food / 1699-Cart / 2371-Hot dog / \n",
"1-Vehicle / 4-Car / 64-Transport / 84-Snow / 223-Weather / 490-Rain / 663-Street / 915-Town / \n",
"14-Guitar / 100-Electric guitar / \n",
"0-Games / 1-Vehicle / 50-Aircraft / 151-Landing / 1715-Aircraft carrier / \n",
"0-Games / 2-Video game / 20-Trailer / 1028-Devil May Cry / 2344-DmC: Devil May Cry / \n",
"5-Dance / \n",
"415-Sewing / 997-Sewing machine / 1479-Quilt / \n",
"1-Vehicle / 50-Aircraft / 139-Airport / 151-Landing / 254-Runway / 856-Airbus A320 family / \n",
"476-Textile / 1184-Flag / \n",
"3-Concert / \n",
"209-University / \n",
"28-Fashion / 337-Advertising / \n",
"1-Vehicle / 27-Motorcycle / 207-Exhaust system / \n",
"56-Hair / 106-Hairstyle / 583-Wig / \n",
"1-Vehicle / 4-Car / \n",
"1525-Wallet / 2068-Duct tape / \n",
"0-Games / 59-Winter sport / 81-Athlete / 112-Ice skating / 142-Hockey / 312-Ice rink / \n",
"242-Paper / 415-Sewing / 1525-Wallet / 2067-Duct / 2068-Duct tape / \n",
"9-Music video / \n",
"246-Furniture / 248-Hotel / 276-Room / \n",
"5-Dance / 9-Music video / \n",
"1-Vehicle / 4-Car / 945-MINI Cooper / 1855-Video camera / \n",
"0-Games / 8-Football / 54-Highlight film / 65-Kick / 76-Ball / \n",
"28-Fashion / 153-Dress / 253-Carnival / 2793-Disguise / \n",
"22-Nature / 61-Art / 148-Painting / 164-Beach / \n",
"819-World Wide Web / \n",
"5-Dance / 9-Music video / \n",
"23-Mobile phone / 29-Smartphone / 553-Sony Xperia / 3019-Sony Ericsson Xperia X10 / \n",
"5-Dance / 16-Performance art / 95-Talent show / \n",
"0-Games / 8-Football / \n",
"10-Animal / 80-Horse / \n",
"0-Games / 8-Football / 54-Highlight film / 81-Athlete / 358-Rugby football / \n",
"1-Vehicle / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 114-Engine / 449-Honda / 552-Honda Civic / \n",
"440-Lion / 1156-Simba / \n",
"843-The Doctor / \n",
"20-Trailer / 125-Television / \n",
"6-Animation / 15-Cartoon / 97-Drawing / 3137-Stitch / \n",
"1-Vehicle / 27-Motorcycle / 337-Advertising / \n",
"3165-Turban / \n",
"0-Games / 8-Football / 76-Ball / \n",
"10-Animal / 48-Pet / 166-Cat / 346-Kitten / \n",
"20-Trailer / 444-Family / 1014-Movieclips / 3996-Fandango / \n",
"0-Games / 269-Comic book / 365-Fireworks / \n",
"0-Games / 6-Animation / 77-Arena / 89-Comics / 123-Naruto / \n",
"0-Games / 2-Video game / 1983-Infestation: Survivor Stories / \n",
"0-Games / 8-Football / 21-Stadium / 73-Ball / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 41-Sports car / 118-Grand Theft Auto V / 330-Money / \n",
"49-School / \n",
"1791-Stargate / \n",
"1-Vehicle / \n",
"25-Toy / 200-Doll / 412-Monster / 503-Monster High / \n",
"319-Robot / \n",
"0-Games / 2-Video game / 1427-TalesRunner / \n",
"85-Combat / \n",
"3-Concert / \n",
"9-Music video / 200-Doll / \n",
"849-Tales / \n",
"3-Concert / 36-Piano / 2120-Tears / \n",
"1-Vehicle / 4-Car / 40-Road / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 2655-Vehicle horn / \n",
"0-Games / 2-Video game / 199-Wii / 426-Resident Evil / \n",
"75-Wrestling / 77-Arena / 777-WWE 2K / 1080-WWE 2K15 / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 6-Animation / \n",
"0-Games / 2-Video game / \n",
"0-Games / 2-Video game / 33-Weapon / 192-Battlefield / 383-Battlefield 4 / 1267-Battlefield Hardline / \n",
"3-Concert / 38-Orchestra / 98-Festival / \n",
"67-Cycling / 69-Bicycle / 154-Wheel / \n",
"260-Prayer / \n",
"83-Skateboarding / 1937-Inline skates / \n",
"23-Mobile phone / 29-Smartphone / 101-Telephone / 131-Computer / 141-Microsoft Windows / 1379-HTC One / 2328-HTC One (M8) / \n",
"45-Cosmetics / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"3-Concert / 14-Guitar / 16-Performance art / \n",
"1-Vehicle / 12-Food / 156-Agriculture / 174-Farm / 931-Combine Harvester / 1158-Wheat / \n",
"3-Concert / 30-Drummer / 39-Drums / 44-Drums / \n",
"1892-Dormitory / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / \n",
"5-Dance / 1100-Hat / \n",
"66-Bollywood / \n",
"6-Animation / 183-Manga / 2357-Vampire Knight / \n",
"9-Music video / \n",
"0-Games / 159-World of Warcraft / \n",
"10-Animal / 12-Food / 26-Cooking / 32-Recipe / 274-Meat / 315-Knife / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 94-Dashcam / 473-Ford Mustang / 552-Honda Civic / 1040-Street racing / \n",
"28-Fashion / 956-Glasses / \n",
"22-Nature / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"9-Music video / \n",
"59-Winter sport / 112-Ice skating / 142-Hockey / \n",
"47-Personal computer / \n",
"3-Concert / 31-Disc jockey / 133-Nightclub / \n",
"0-Games / 2-Video game / 47-Personal computer / 429-Website / \n",
"0-Games / 637-Five Nights at Freddy's / \n",
"22-Nature / 147-River / 283-Rock / 1416-Cliff / \n",
"22-Nature / 86-Plant / 376-Flute / 2532-Pan flute / \n",
"3-Concert / \n",
"9-Music video / \n",
"10-Animal / 131-Computer / \n",
"122-Weight training / 204-Gym / 214-Muscle / \n",
"0-Games / 2-Video game / 23-Mobile phone / 29-Smartphone / 72-iPhone / 331-iPod touch / \n",
"9-Music video / 78-Wedding / 190-Bride / \n",
"3-Concert / 49-School / \n",
"10-Animal / 68-Lighting / 91-Fish / 258-Aquarium / 716-Coral / 1051-Reef aquarium / \n",
"323-Toddler / \n",
"2-Video game / 9-Music video / 2173-Dead Space / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / 40-Road / 64-Transport / 105-Motorcycling / 154-Wheel / 482-Asphalt / 608-Wheelie / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 157-Jet aircraft / \n",
"12-Food / 20-Trailer / \n",
"6-Animation / 15-Cartoon / 183-Manga / 762-Angel / \n",
"256-Tennis / \n",
"0-Games / 2-Video game / 6-Animation / 35-Minecraft / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 169-Rallying / \n",
"33-Weapon / 217-Hunting / \n",
"638-Gears of War / 773-Gears of War / 1425-Gears of War 2 / \n",
"0-Games / 2-Video game / 33-Weapon / 229-Halo / 470-Halo 3 / \n",
"3938-Large Hadron Collider / \n",
"0-Games / 6-Animation / 89-Comics / \n",
"0-Games / 8-Football / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 39-Drums / 763-Snake / \n",
"45-Cosmetics / 890-Brush / 3655-Makeup brush / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 92-Radio-controlled model / 135-Drifting / 202-Radio-controlled car / \n",
"3-Concert / \n",
"7-Musician / 9-Music video / 13-Musical ensemble / 14-Guitar / 30-Drummer / 44-Drums / \n",
"0-Games / 8-Football / 54-Highlight film / \n",
"0-Games / 2-Video game / 33-Weapon / 96-Soldier / 834-Combat Arms / \n",
"12-Food / 88-Machine / 227-Water / 938-Steel / \n",
"1-Vehicle / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 172-Motocross / 769-Red Bull / \n",
"1-Vehicle / 4-Car / 41-Sports car / 245-BMW / 344-Coupé / 493-Rim / 1233-BMW 3 Series / 2325-BMW 3 Series (E90) / \n",
"1-Vehicle / 4-Car / 1742-Lowrider / 3040-Subaru Legacy / \n",
"3-Concert / 7-Musician / \n",
"205-Newscaster / \n",
"1-Vehicle / 4-Car / 9-Music video / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / 111-PlayStation 3 / 127-Xbox 360 / 176-PlayStation 4 / 181-Xbox / 314-Xbox One / 843-The Doctor / \n",
"10-Animal / 22-Nature / 42-Fishing / 91-Fish / 147-River / 814-Pond / \n",
"22-Nature / 36-Piano / 150-Violin / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 378-Drag racing / \n",
"0-Games / 257-Counter-Strike / 1109-Counter-Strike Online / \n",
"0-Games / 2-Video game / 1187-Ragnarok Online / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 111-PlayStation 3 / 1492-Persona / \n",
"7-Musician / 14-Guitar / 738-Guitar amplifier / \n",
"3-Concert / \n",
"47-Personal computer / 72-iPhone / 145-Tablet computer / 189-iPad / \n",
"275-Diving / 287-Human swimming / \n",
"12-Food / 28-Fashion / 45-Cosmetics / 56-Hair / 106-Hairstyle / 909-Updo / \n",
"3-Concert / 46-Choir / 318-Church / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / 472-Boeing 737 / 499-Cloud / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 217-Hunting / 706-Deer / \n",
"8-Football / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / \n",
"5-Dance / 60-Basketball / 191-Cheerleading / 3960-Pep squad / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 116-Pokémon / 186-Pokémon / 690-Pokémon X and Y / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"3-Concert / \n",
"747-Retail / \n",
"1-Vehicle / 50-Aircraft / 175-Airliner / 188-Airline / 1340-Airbus A380 / 2587-Air France / \n",
"122-Weight training / 214-Muscle / \n",
"25-Toy / 295-Chocolate / 1192-Hello Kitty / \n",
"0-Games / 2-Video game / 33-Weapon / 257-Counter-Strike / 702-Counter-Strike: Source / 826-Source Engine / \n",
"0-Games / 2-Video game / 111-PlayStation 3 / \n",
"23-Mobile phone / 29-Smartphone / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 398-The Legend of Zelda / 2442-The Legend of Zelda: The Wind Waker / \n",
"59-Winter sport / 112-Ice skating / 142-Hockey / 262-Ice / \n",
"1-Vehicle / 4-Car / 74-Truck / 84-Snow / 167-Four-wheel drive / 198-Off-road vehicle / 277-Sport utility vehicle / 327-Mud / 360-Jeep / \n",
"1-Vehicle / 62-Train / 840-Museum / \n",
"3-Concert / 5-Dance / 10-Animal / 16-Performance art / 80-Horse / \n",
"10-Animal / \n",
"121-Photography / \n",
"1-Vehicle / 4-Car / 6-Animation / 809-Logo / \n",
"1-Vehicle / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 266-Helicopter / 392-Radio-controlled helicopter / 2867-Wasp / \n",
"0-Games / 2-Video game / 1312-Dynasty Warriors / \n",
"3-Concert / 7-Musician / 16-Performance art / 38-Orchestra / \n",
"0-Games / 211-RuneScape / \n",
"0-Games / 8-Football / 90-Sports game / 453-Pro Evolution Soccer / 1719-Pro Evolution Soccer 2015 / \n",
"0-Games / 35-Minecraft / 1149-Castle / \n",
"0-Games / 1680-Seafight / \n",
"3-Concert / 16-Performance art / 98-Festival / \n",
"1791-Stargate / \n",
"1-Vehicle / 94-Dashcam / 102-Building / 416-Apartment / 915-Town / 1960-Traffic light / \n",
"3-Concert / \n",
"31-Disc jockey / \n",
"1-Vehicle / 4-Car / 41-Sports car / 70-Driving / 234-Ford / 473-Ford Mustang / 1361-Ford GT / 1388-V6 engine / \n",
"3-Concert / 16-Performance art / \n",
"0-Games / 2-Video game / 4140-Quake Live / \n",
"365-Fireworks / \n",
"81-Athlete / 163-Running / \n",
"6-Animation / \n",
"28-Fashion / 205-Newscaster / 1386-ABS-CBN / \n",
"3-Concert / 30-Drummer / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 297-Grand Theft Auto IV / 658-Grand Theft Auto: The Lost and Damned / 1134-Taxicab / 1307-Police car / \n",
"253-Carnival / \n",
"0-Games / 2-Video game / 229-Halo / 470-Halo 3 / \n",
"5-Dance / 16-Performance art / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 2863-Max Payne 3 / \n",
"0-Games / 2-Video game / 19-PC game / 686-Diablo III / 872-Diablo / \n",
"9-Music video / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 301-Meal / \n",
"0-Games / 82-Airplane / 506-Garry's Mod / \n",
"3-Concert / \n",
"56-Hair / \n",
"6-Animation / 15-Cartoon / 440-Lion / 1212-Stuffed toy / \n",
"307-Hand / \n",
"22-Nature / 113-House / 208-Wood / 267-Mountain / 2280-Cottage / \n",
"78-Wedding / 190-Bride / 421-Gown / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 16-Performance art / 24-String instrument / \n",
"0-Games / 33-Weapon / 474-Destiny / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 261-Tool / \n",
"116-Pokémon / 186-Pokémon / 690-Pokémon X and Y / \n",
"5-Dance / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 70-Driving / 207-Exhaust system / 509-Classic car / \n",
"0-Games / 2-Video game / \n",
"49-School / \n",
"1-Vehicle / 4-Car / 234-Ford / 3338-Ford Crown Victoria / \n",
"0-Games / 54-Highlight film / 81-Athlete / \n",
"3-Concert / 98-Festival / 253-Carnival / \n",
"0-Games / 75-Wrestling / 85-Combat / 2822-Submission wrestling / \n",
"1-Vehicle / 11-Motorsport / 18-Outdoor recreation / 22-Nature / 327-Mud / 565-Mud bogging / \n",
"3-Concert / 16-Performance art / \n",
"0-Games / 2-Video game / 116-Pokémon / 186-Pokémon / \n",
"5-Dance / 130-Gymnastics / 350-Circus / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 210-Cookware and bakeware / 239-Roasting / 391-Chicken meat / 1284-Frying / 2563-Gravy / 2591-Filipino food / 2757-Fried chicken / \n",
"0-Games / 8-Football / \n",
"6-Animation / 123-Naruto / 285-Sasuke Uchiha / \n",
"1-Vehicle / 50-Aircraft / 129-Aviation / 157-Jet aircraft / 388-Fighter aircraft / 758-Air force / 2326-Royal Air Force / \n",
"0-Games / 2-Video game / 47-Personal computer / 131-Computer / 228-Laptop / 309-Computer hardware / 602-Central processing unit / \n",
"1-Vehicle / 18-Outdoor recreation / 87-Boat / 579-Yacht / \n",
"1-Vehicle / 4-Car / 40-Road / 158-Tire / 167-Four-wheel drive / 277-Sport utility vehicle / 403-Toyota / 1511-Toyota Land Cruiser / \n",
"447-Sand / \n",
"21-Stadium / 49-School / 562-Pitcher / \n",
"36-Piano / 53-Keyboard / 541-Black-and-white / \n",
"15-Cartoon / 1134-Taxicab / \n",
"0-Games / 2-Video game / 685-Luigi / 3900-Mario & Luigi: Superstar Saga / \n",
"1-Vehicle / 4-Car / 270-Sedan / 1074-Hyundai / 3316-Hyundai Genesis / \n",
"0-Games / 2-Video game / 8-Football / 90-Sports game / 180-FIFA 15 / \n",
"117-Boxing / \n",
"42-Fishing / 510-Fishing lure / \n",
"0-Games / 2-Video game / 61-Art / \n",
"1-Vehicle / 6-Animation / 25-Toy / 140-LEGO / 1079-Lego City / 1302-The Lego Group / \n",
"341-Sketch comedy / \n",
"219-Accordion / 655-Diatonic button accordion / \n",
"5-Dance / 126-The Walt Disney Company / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 349-Chevrolet / 406-Chevrolet / 1148-Hot rod / \n",
"162-Medicine / \n",
"61-Art / 148-Painting / 628-Portrait / \n",
"1572-Husband / \n",
"0-Games / 2-Video game / 2799-Ys / \n",
"31-Disc jockey / \n",
"5-Dance / 115-Ballet / \n",
"85-Combat / 117-Boxing / \n",
"31-Disc jockey / \n",
"14-Guitar / \n",
"1-Vehicle / 50-Aircraft / 187-Takeoff / 254-Runway / 266-Helicopter / 4657-Bell Boeing V-22 Osprey / \n",
"6-Animation / 9-Music video / 1160-Giant panda / \n",
"1-Vehicle / 666-Concrete / \n",
"5-Dance / \n",
"609-Tattoo / \n",
"45-Cosmetics / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"6-Animation / 15-Cartoon / \n",
"191-Cheerleading / 776-Kindergarten / \n",
"46-Choir / \n",
"2-Video game / 20-Trailer / \n",
"6-Animation / 15-Cartoon / \n",
"36-Piano / 3117-Match / 3352-Helianthus annuus / \n",
"0-Games / 2-Video game / 489-Board game / \n",
"0-Games / 2-Video game / 19-PC game / 523-Cattle / 686-Diablo III / \n",
"9-Music video / \n",
"3-Concert / \n",
"3-Concert / 7-Musician / \n",
"125-Television / 355-Comedian / \n",
"1-Vehicle / 4-Car / 114-Engine / 261-Tool / 585-Lawn / 741-Mower / 805-Lawn mower / \n",
"504-Web page / \n",
"12-Food / 215-Eating / 762-Angel / \n",
"3-Concert / \n",
"59-Winter sport / 112-Ice skating / 142-Hockey / \n",
"208-Wood / 400-Woodturning / 972-Lathe / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"31-Disc jockey / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 105-Motorcycling / 212-Highway / \n",
"6-Animation / 15-Cartoon / \n",
"6-Animation / 15-Cartoon / 61-Art / 97-Drawing / 183-Manga / 408-Sketch / \n",
"10-Animal / 132-Bird / 217-Hunting / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / 169-Rallying / \n",
"47-Personal computer / 497-Macintosh / 1922-iMac / 3974-Mac Mini / \n",
"7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 122-Weight training / 179-Kitchen / 210-Cookware and bakeware / 742-Countertop / 2027-Spinach / 2066-Omelette / 2309-Low-carbohydrate diet / 3150-Supper / \n",
"1868-Preacher / \n",
"121-Photography / 155-Camera / \n",
"38-Orchestra / 290-Pony / 343-My Little Pony / 1319-Apple / 4042-Applejack / \n",
"3-Concert / 98-Festival / \n",
"3085-Charango / \n",
"0-Games / 326-Metin2 / \n",
"0-Games / 19-PC game / 35-Minecraft / \n",
"148-Painting / 208-Wood / \n",
"8-Football / 21-Stadium / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 603-StarCraft II: Wings of Liberty / 732-StarCraft / \n",
"0-Games / 8-Football / 21-Stadium / 60-Basketball / \n",
"12-Food / 533-Oil / \n",
"31-Disc jockey / \n",
"5-Dance / 31-Disc jockey / \n",
"8-Football / \n",
"28-Fashion / 333-Nike; Inc. / 1795-Sock / \n",
"33-Weapon / 315-Knife / \n",
"130-Gymnastics / \n",
"0-Games / 2-Video game / 43-Call of Duty / 134-Call of Duty: Black Ops / \n",
"3-Concert / \n",
"98-Festival / 448-Planet / \n",
"45-Cosmetics / 282-Skin / 2351-Moisturizer / 3861-Toner / \n",
"18-Outdoor recreation / 59-Winter sport / 84-Snow / 120-Winter / 177-Skiing / 267-Mountain / 305-Ski / 789-Alpine skiing / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / \n",
"5-Dance / 115-Ballet / \n",
"214-Muscle / \n",
"369-Resort / \n",
"12-Food / 26-Cooking / 32-Recipe / 109-Cooking show / 226-Dessert / 232-Cake / 236-Baking / 560-Sugar / 1641-Cheesecake / \n",
"23-Mobile phone / 29-Smartphone / 47-Personal computer / 72-iPhone / 145-Tablet computer / 189-iPad / \n",
"79-American football / 209-University / \n",
"12-Food / 86-Plant / 136-Gardening / 137-Tree / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 85-Combat / 1166-Dark Souls II / \n",
"4303-Apple Store / \n",
"56-Hair / 106-Hairstyle / \n",
"10-Animal / 1108-Mushroom / \n",
"1-Vehicle / 11-Motorsport / 565-Mud bogging / \n",
"448-Planet / 654-Sun / 1702-Telescope / 2761-Venus / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / 1904-Electric locomotive / \n",
"1-Vehicle / 4-Car / 23-Mobile phone / 234-Ford / 1634-Ford Fiesta / \n",
"1-Vehicle / 25-Toy / 126-The Walt Disney Company / \n",
"7-Musician / 9-Music video / 14-Guitar / 627-Clock / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 192-Battlefield / 383-Battlefield 4 / \n",
"0-Games / 51-Strategy video game / 165-League of Legends / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / \n",
"148-Painting / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 42-Fishing / 91-Fish / 275-Diving / 288-Underwater / 381-Underwater diving / 407-Scuba diving / 716-Coral / \n",
"30-Drummer / 39-Drums / 44-Drums / \n",
"3-Concert / 7-Musician / 16-Performance art / 30-Drummer / 279-Garden / 4329-Busch Gardens Tampa Bay / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / \n",
"983-Lexus / \n",
"1-Vehicle / 27-Motorcycle / 336-Scooter / \n",
"1858-Poster / \n",
"260-Prayer / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / 1971-Epiphone / \n",
"634-Trombone / \n",
"0-Games / 1-Vehicle / 4-Car / 11-Motorsport / 70-Driving / 837-Roblox / \n",
"3-Concert / 9-Music video / 13-Musical ensemble / \n",
"222-Surfing / 1008-Kite / \n",
"0-Games / 211-RuneScape / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / 734-DayZ / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"139-Airport / 159-World of Warcraft / 171-Warcraft / \n",
"0-Games / 23-Mobile phone / 29-Smartphone / 72-iPhone / \n",
"653-Thread / 1497-Wool / \n",
"0-Games / 211-RuneScape / \n",
"5-Dance / 78-Wedding / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 157-Jet aircraft / 388-Fighter aircraft / 758-Air force / 929-Aerobatics / \n",
"9-Music video / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 257-Counter-Strike / 315-Knife / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"12-Food / 88-Machine / 227-Water / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / 506-Garry's Mod / 2290-Trouble in Terrorist Town / \n",
"0-Games / 75-Wrestling / 777-WWE 2K / 977-WWE '13 / 1582-WWE 2K14 / \n",
"322-Mixtape / \n",
"88-Machine / 3270-Jukebox / \n",
"9-Music video / 541-Black-and-white / \n",
"31-Disc jockey / \n",
"93-Comedy / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 111-PlayStation 3 / 456-Go-kart / 711-Gran Turismo / 769-Red Bull / 1519-Gran Turismo 6 / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"111-PlayStation 3 / 849-Tales / \n",
"0-Games / 2-Video game / 25-Toy / 140-LEGO / \n",
"5-Dance / 16-Performance art / \n",
"1-Vehicle / 87-Boat / \n",
"10-Animal / 22-Nature / 132-Bird / 137-Tree / 213-Forest / 217-Hunting / 280-Wildlife / 2333-Wild turkey / 4351-Turkey / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 16-Performance art / 30-Drummer / 44-Drums / \n",
"12-Food / 28-Fashion / 232-Cake / 236-Baking / \n",
"20-Trailer / \n",
"24-String instrument / 150-Violin / 2477-Electric violin / \n",
"0-Games / 8-Football / 21-Stadium / 73-Ball / \n",
"28-Fashion / 121-Photography / \n",
"3-Concert / 68-Lighting / \n",
"14-Guitar / 293-Slide show / \n",
"1-Vehicle / 62-Train / 64-Transport / \n",
"419-Earth / 593-Moon / 654-Sun / 2022-Eclipse / 2741-Solar eclipse / 3919-Lunar eclipse / \n",
"102-Building / 248-Hotel / \n",
"0-Games / 8-Football / \n",
"1-Vehicle / 4-Car / 332-Mercedes-Benz / 3826-Mercedes-Benz A-Class / \n",
"0-Games / 2-Video game / \n",
"1-Vehicle / 4-Car / 64-Transport / \n",
"5-Dance / \n",
"36-Piano / 107-Musical keyboard / 265-Electronic keyboard / \n",
"1-Vehicle / 1257-Turbine / \n",
"1-Vehicle / 87-Boat / \n",
"7-Musician / 13-Musical ensemble / 38-Orchestra / 150-Violin / \n",
"25-Toy / 200-Doll / 269-Comic book / 290-Pony / 343-My Little Pony / \n",
"12-Food / 162-Medicine / 780-Fat / \n",
"616-The Bible / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / 4159-Grease / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"1066-Aikido / \n",
"44-Drums / 376-Flute / \n",
"9-Music video / 30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"365-Fireworks / \n",
"60-Basketball / \n",
"840-Museum / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 87-Boat / 169-Rallying / \n",
"0-Games / 2-Video game / 791-Magic: The Gathering / \n",
"1-Vehicle / 50-Aircraft / 87-Boat / 147-River / 266-Helicopter / 308-Ship / 1107-United States Navy / \n",
"2013-The Twilight Saga / \n",
"98-Festival / \n",
"0-Games / 2-Video game / 19-PC game / 596-Monster Hunter / 1339-Monster Hunter Freedom Unite / \n",
"3-Concert / 5-Dance / 16-Performance art / 115-Ballet / \n",
"34-Action-adventure game / 201-Grand Theft Auto: San Andreas / \n",
"1-Vehicle / 4-Car / 41-Sports car / 68-Lighting / 92-Radio-controlled model / 202-Radio-controlled car / 245-BMW / 1418-Model car / \n",
"5-Dance / 6-Animation / 15-Cartoon / 2408-Lucky Star / \n",
"3-Concert / 7-Musician / 46-Choir / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 165-League of Legends / 1494-Jungle / \n",
"0-Games / 8-Football / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 135-Drifting / \n",
"3-Concert / \n",
"144-Amusement park / \n",
"75-Wrestling / \n",
"6-Animation / 89-Comics / 183-Manga / \n",
"1-Vehicle / 4-Car / 447-Sand / 751-Desert / 853-Dune buggy / 975-Dune / \n",
"9-Music video / 14-Guitar / \n",
"425-City / \n",
"22-Nature / \n",
"7-Musician / 14-Guitar / 100-Electric guitar / 738-Guitar amplifier / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 77-Arena / 81-Athlete / \n",
"294-Batman / 838-Batman: Arkham / 1881-Arkham Asylum / 2084-Batman: Arkham Origins / 2146-Bane / \n",
"10-Animal / 550-Dinosaur / 1126-Tyrannosaurus / \n",
"10-Animal / 18-Outdoor recreation / 132-Bird / 718-Duck / \n",
"9-Music video / \n",
"260-Prayer / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 454-Sauce / 752-Pasta / 1517-Spaghetti / \n",
"12-Food / 26-Cooking / 539-Home appliance / 649-Oven / 943-Stove / 2490-Kitchen stove / 4586-Hotpoint / \n",
"3-Concert / 7-Musician / 24-String instrument / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 55-Video game console / 176-PlayStation 4 / 317-PlayStation / \n",
"6-Animation / 10-Animal / 15-Cartoon / 290-Pony / 343-My Little Pony / \n",
"85-Combat / 117-Boxing / 399-Kickboxing / \n",
"75-Wrestling / \n",
"10-Animal / 17-Racing / 18-Outdoor recreation / 1117-Camcorder / 2883-Flip Video / \n",
"10-Animal / 22-Nature / 132-Bird / 213-Forest / 217-Hunting / 280-Wildlife / \n",
"6-Animation / 123-Naruto / 285-Sasuke Uchiha / \n",
"0-Games / 2-Video game / 237-Final Fantasy / 1571-Final Fantasy XIV / 1760-Final Fantasy XIV: A Realm Reborn / \n",
"42-Fishing / 222-Surfing / 672-Bodyboarding / \n",
"0-Games / 586-Nintendo Entertainment System / \n",
"1-Vehicle / 4-Car / 260-Prayer / \n",
"325-Vampire / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / \n",
"1-Vehicle / 4-Car / 270-Sedan / 1074-Hyundai / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 70-Driving / \n",
"22-Nature / \n",
"2-Video game / 20-Trailer / 3159-Dishonored / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / \n",
"238-Nail / 304-Nail polish / 1424-Coupon / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / 90-Sports game / 561-Madden NFL / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / 99-Christmas / \n",
"1-Vehicle / 4-Car / 299-Volkswagen Passenger Cars / 531-Hatchback / 1661-Volkswagen Gol / \n",
"22-Nature / 121-Photography / 155-Camera / \n",
"18-Outdoor recreation / 458-Gold / 1426-Treasure / \n",
"59-Winter sport / 112-Ice skating / 316-Figure skating / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 111-PlayStation 3 / 1106-Character / 1312-Dynasty Warriors / 3721-Dynasty Warriors 7 / 3813-Dynasty Warriors: Gundam / \n",
"38-Orchestra / \n",
"0-Games / 2-Video game / 6-Animation / 2237-Kantai Collection / \n",
"1-Vehicle / 4-Car / 270-Sedan / 2509-Mazda3 / \n",
"3-Concert / 38-Orchestra / 68-Lighting / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / \n",
"31-Disc jockey / \n",
"67-Cycling / 69-Bicycle / 377-BMX bike / \n",
"0-Games / 7-Musician / 14-Guitar / \n",
"1-Vehicle / 103-Rail transport / 224-Rapid transit / \n",
"113-House / 170-Home improvement / 246-Furniture / 416-Apartment / 580-Bedroom / \n",
"6-Animation / 379-Touhou Project / \n",
"93-Comedy / \n",
"1-Vehicle / 25-Toy / 50-Aircraft / 92-Radio-controlled model / 149-Model aircraft / 155-Camera / 160-Radio-controlled aircraft / 266-Helicopter / 392-Radio-controlled helicopter / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 178-Samsung Galaxy / 883-Samsung Galaxy S4 / 1015-Samsung Galaxy Note series / 1776-Samsung Galaxy Note II / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / 1127-Drum stick / \n",
"5-Dance / 46-Choir / 209-University / \n",
"347-Climbing / \n",
"12-Food / 26-Cooking / 313-Book / \n",
"341-Sketch comedy / 364-Sitcom / \n",
"3-Concert / 68-Lighting / \n",
"694-Tai chi / \n",
"0-Games / 2-Video game / 1290-Dragon Nest / \n",
"12-Food / 28-Fashion / 476-Textile / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 42-Fishing / 91-Fish / 225-Lake / 814-Pond / \n",
"1-Vehicle / 4-Car / 769-Red Bull / \n",
"130-Gymnastics / 1417-Balance beam / \n",
"694-Tai chi / \n",
"3-Concert / \n",
"8-Football / 710-Handball / \n",
"3-Concert / 16-Performance art / 30-Drummer / 39-Drums / 44-Drums / \n",
"14-Guitar / 606-String / 670-Banjo / \n",
"10-Animal / 17-Racing / 57-Race track / 80-Horse / 384-Horse racing / \n",
"1-Vehicle / 4-Car / 299-Volkswagen Passenger Cars / 526-Volkswagen / 969-Volkswagen Beetle / \n",
"0-Games / 79-American football / 993-Quarterback / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / \n",
"14-Guitar / 36-Piano / 99-Christmas / \n",
"227-Water / 278-GoPro / 287-Human swimming / 288-Underwater / 329-Swimming pool / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 389-Restaurant / \n",
"125-Television / \n",
"89-Comics / 483-Costume / 803-DC Comics / \n",
"144-Amusement park / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / 1127-Drum stick / \n",
"1-Vehicle / 4-Car / 40-Road / 449-Honda / 4099-Honda Prelude / \n",
"12-Food / 26-Cooking / 32-Recipe / 1329-Blender / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 42-Fishing / 91-Fish / 814-Pond / \n",
"33-Weapon / 261-Tool / 315-Knife / \n",
"163-Running / \n",
"12-Food / \n",
"136-Gardening / 1384-Greenhouse / \n",
"25-Toy / 290-Pony / 343-My Little Pony / 524-Littlest Pet Shop / \n",
"6-Animation / 15-Cartoon / 1156-Simba / 3261-Princess Jasmine / 3600-The Jungle Book / 4202-Timon and Pumbaa / \n",
"1-Vehicle / 74-Truck / 433-Construction / 4339-Deep foundation / \n",
"748-Iron Man / 897-Avengers / 1173-Captain America / \n",
"111-PlayStation 3 / 420-The Elder Scrolls V: Skyrim / 924-Trophy / \n",
"163-Running / \n",
"0-Games / 2-Video game / 6-Animation / 311-Street Fighter / 514-Tekken / 3634-Juri / 3740-Pandora / \n",
"12-Food / 26-Cooking / 274-Meat / 589-Grilling / 620-Barbecue / 1085-Barbecue grill / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"1-Vehicle / 4-Car / 683-Renault / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"0-Games / 2-Video game / 345-Loudspeaker / \n",
"6-Animation / 25-Toy / 200-Doll / 390-Model / 439-Barbie / \n",
"75-Wrestling / \n",
"427-Trumpet / \n",
"1091-Saints Row / 1695-Saints Row: The Third / \n",
"0-Games / 2-Video game / 43-Call of Duty / 111-PlayStation 3 / 125-Television / 134-Call of Duty: Black Ops / 2786-3D television / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 138-Call of Duty: Black Ops II / 422-Call of Duty: Ghosts / \n",
"46-Choir / \n",
"1-Vehicle / 4-Car / 299-Volkswagen Passenger Cars / 526-Volkswagen / 1761-Volkswagen Jetta / 3196-DVD player / \n",
"318-Church / 885-Bell / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / 397-Organ / \n",
"471-Wing / 3333-Cavalry / \n",
"3114-Surveillance / \n",
"3-Concert / 68-Lighting / \n",
"23-Mobile phone / 29-Smartphone / \n",
"3-Concert / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 99-Christmas / 1332-Onion / \n",
"49-School / \n",
"59-Winter sport / 112-Ice skating / 316-Figure skating / 632-Ice dancing / \n",
"1-Vehicle / 4-Car / 64-Transport / 74-Truck / \n",
"28-Fashion / 45-Cosmetics / 56-Hair / 2235-Avon Products / \n",
"45-Cosmetics / \n",
"0-Games / 2-Video game / 8-Football / 90-Sports game / 180-FIFA 15 / 480-FIFA 13 / 848-FIFA 12 / \n",
"49-School / \n",
"730-Final Fantasy VII / \n",
"59-Winter sport / 84-Snow / 177-Skiing / 281-Snowboarding / 305-Ski / 501-Snowboard / \n",
"3-Concert / \n",
"56-Hair / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 299-Volkswagen Passenger Cars / 526-Volkswagen / 969-Volkswagen Beetle / \n",
"18-Outdoor recreation / 33-Weapon / 76-Ball / 3264-Golf ball / \n",
"8-Football / \n",
"162-Medicine / \n",
"222-Surfing / 672-Bodyboarding / \n",
"12-Food / 1060-Raw food / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 6-Animation / 241-Fighting game / 486-M.U.G.E.N / 764-The King of Fighters / 1988-Chun-Li / \n",
"0-Games / 116-Pokémon / 186-Pokémon / 1005-Pokémon Omega Ruby and Alpha Sapphire / \n",
"6-Animation / 25-Toy / \n",
"2-Video game / 6-Animation / \n",
"18-Outdoor recreation / 22-Nature / 1254-Cave / \n",
"0-Games / 2-Video game / 8-Football / 21-Stadium / 90-Sports game / 362-PlayStation Portable / 561-Madden NFL / \n",
"7-Musician / 13-Musical ensemble / 49-School / 203-Marching band / 306-High school / 1675-Drum and bugle corps / \n",
"0-Games / 792-DarkOrbit / \n",
"14-Guitar / \n",
"0-Games / 12-Food / 662-Pizza / \n",
"593-Moon / \n",
"12-Food / \n",
"0-Games / 2-Video game / 55-Video game console / 413-PlayStation 2 / \n",
"45-Cosmetics / 238-Nail / 300-Nail art / 304-Nail polish / 307-Hand / 324-Finger / 338-Manicure / \n",
"1-Vehicle / 2616-Chevrolet S-10 / \n",
"1-Vehicle / 4-Car / 277-Sport utility vehicle / 299-Volkswagen Passenger Cars / 526-Volkswagen / 982-V8 engine / 3587-Volkswagen Touareg / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / 49-School / \n",
"9-Music video / \n",
"3-Concert / 13-Musical ensemble / 38-Orchestra / \n",
"3-Concert / 7-Musician / 16-Performance art / 24-String instrument / \n",
"6-Animation / 15-Cartoon / \n",
"36-Piano / 237-Final Fantasy / 730-Final Fantasy VII / \n",
"469-Bag / 653-Thread / 2329-Felt / \n",
"25-Toy / 1052-Yo-Kai Watch / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 25-Toy / 3602-Mini-Z / \n",
"677-Game controller / 1046-Remote control / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 434-Ninja / 2433-Trials / 3723-Trials Evolution / \n",
"0-Games / 2-Video game / 47-Personal computer / 294-Batman / 838-Batman: Arkham / 1078-Installation art / 2084-Batman: Arkham Origins / \n",
"9-Music video / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 30-Drummer / 39-Drums / 3162-Postage stamp / \n",
"121-Photography / 155-Camera / 469-Bag / 835-Backpack / \n",
"322-Mixtape / \n",
"1-Vehicle / 4-Car / 349-Chevrolet / 3353-Chevrolet Monte Carlo / 3371-Airbag / \n",
"56-Hair / 340-Afro-textured hair / 583-Wig / 607-Lace / \n",
"29-Smartphone / \n",
"5-Dance / 6-Animation / \n",
"7-Musician / 219-Accordion / 655-Diatonic button accordion / \n",
"294-Batman / 4701-Scarecrow / \n",
"1-Vehicle / 4-Car / 484-Subwoofer / \n",
"0-Games / 35-Minecraft / 625-Factory / 1913-Solar panel / \n",
"81-Athlete / 562-Pitcher / \n",
"46-Choir / \n",
"14-Guitar / 2738-Fender Jazz Bass / \n",
"8-Football / 767-Goalkeeper / \n",
"3-Concert / 16-Performance art / \n",
"2-Video game / 116-Pokémon / 186-Pokémon / 1153-Pokémon HeartGold and SoulSilver / \n",
"3-Concert / 5-Dance / 16-Performance art / 49-School / 306-High school / \n",
"42-Fishing / 164-Beach / 222-Surfing / 233-Ocean / 672-Bodyboarding / \n",
"5-Dance / \n",
"0-Games / 60-Basketball / 168-Basketball moves / 255-Slam dunk / 1608-Injury / \n",
"6-Animation / 232-Cake / 2313-Bánh / \n",
"0-Games / 257-Counter-Strike / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"88-Machine / \n",
"110-Album / \n",
"15-Cartoon / 61-Art / 97-Drawing / 366-Dragon / \n",
"4300-Tambourine / \n",
"5-Dance / 7-Musician / 16-Performance art / 99-Christmas / \n",
"6-Animation / 15-Cartoon / \n",
"20-Trailer / \n",
"1183-Karaoke box / \n",
"3-Concert / 16-Performance art / \n",
"6-Animation / 15-Cartoon / \n",
"9-Music video / \n",
"5-Dance / \n",
"25-Toy / 200-Doll / 962-McDonald's / \n",
"3-Concert / 7-Musician / 38-Orchestra / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / \n",
"0-Games / 2-Video game / 55-Video game console / 181-Xbox / \n",
"2-Video game / 25-Toy / 126-The Walt Disney Company / 1106-Character / 2453-Disney Infinity / \n",
"130-Gymnastics / 350-Circus / \n",
"3-Concert / 68-Lighting / \n",
"0-Games / 494-Card game / 615-Poker / \n",
"85-Combat / 1239-K-1 / \n",
"0-Games / 2-Video game / 37-Gadget / 47-Personal computer / 516-Headphones / 832-Headset / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / \n",
"45-Cosmetics / 195-Mascara / 641-Mask / \n",
"3-Concert / 7-Musician / \n",
"110-Album / \n",
"8-Football / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 88-Machine / 2623-Hose / \n",
"0-Games / 49-School / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 81-Athlete / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 105-Motorcycling / 172-Motocross / \n",
"0-Games / 2-Video game / 21-Stadium / 79-American football / 90-Sports game / 561-Madden NFL / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"28-Fashion / 45-Cosmetics / 56-Hair / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 235-Lipstick / 291-Rouge / 325-Vampire / 385-Face / 436-Concealer / 3655-Makeup brush / \n",
"0-Games / 8-Football / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 620-Barbecue / 1186-Seafood / 2213-Shrimp / 3379-Prawn / \n",
"0-Games / 2-Video game / \n",
"10-Animal / 18-Outdoor recreation / 42-Fishing / 91-Fish / 268-Recreational fishing / \n",
"93-Comedy / \n",
"14-Guitar / 24-String instrument / 670-Banjo / \n",
"1-Vehicle / 50-Aircraft / 139-Airport / 151-Landing / 187-Takeoff / 188-Airline / 254-Runway / \n",
"85-Combat / 399-Kickboxing / 623-Coach / \n",
"1-Vehicle / 4-Car / 270-Sedan / 299-Volkswagen Passenger Cars / 526-Volkswagen / 1761-Volkswagen Jetta / \n",
"3-Concert / 7-Musician / 16-Performance art / 30-Drummer / \n",
"0-Games / 2-Video game / 35-Minecraft / 1176-Pumpkin / \n",
"6-Animation / 15-Cartoon / 185-Dragon Ball / 244-Goku / 418-Gohan / 1304-Cell / \n",
"93-Comedy / 341-Sketch comedy / \n",
"0-Games / 2-Video game / 55-Video game console / 586-Nintendo Entertainment System / \n",
"3-Concert / 7-Musician / 9-Music video / \n",
"222-Surfing / 233-Ocean / 672-Bodyboarding / \n",
"79-American football / \n",
"75-Wrestling / \n",
"0-Games / 2-Video game / 6-Animation / 123-Naruto / 591-Naruto: Ultimate Ninja / \n",
"3-Concert / \n",
"0-Games / 35-Minecraft / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"809-Logo / 3233-Screencast / \n",
"337-Advertising / 2032-Pepsi / \n",
"3-Concert / 7-Musician / \n",
"22-Nature / 425-City / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / \n",
"15-Cartoon / 379-Touhou Project / \n",
"1107-United States Navy / \n",
"0-Games / 2-Video game / 33-Weapon / 1382-Half-Life 2 / 1412-Half-Life / \n",
"0-Games / 19-PC game / 2043-Mabinogi / \n",
"44-Drums / \n",
"12-Food / 295-Chocolate / 1222-Fountain / 2256-Bearing / \n",
"0-Games / 2-Video game / \n",
"150-Violin / 545-Quartet (ensemble) / 1002-String quartet / 2477-Electric violin / \n",
"162-Medicine / 288-Underwater / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 109-Cooking show / 184-Vegetable / 210-Cookware and bakeware / 1114-Stuffing / 4053-Calabaza / \n",
"0-Games / 8-Football / 90-Sports game / 180-FIFA 15 / \n",
"99-Christmas / \n",
"0-Games / 19-PC game / 664-CrossFire / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 302-Call of Duty 4: Modern Warfare / \n",
"8-Football / \n",
"8-Football / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 74-Truck / 167-Four-wheel drive / 198-Off-road vehicle / 327-Mud / 565-Mud bogging / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 24-String instrument / 38-Orchestra / 150-Violin / 1710-Pirates of the Caribbean / \n",
"0-Games / 2-Video game / \n",
"1-Vehicle / 11-Motorsport / 18-Outdoor recreation / 27-Motorcycle / \n",
"0-Games / 51-Strategy video game / 165-League of Legends / \n",
"21-Stadium / 54-Highlight film / \n",
"1-Vehicle / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 393-Unmanned aerial vehicle / 604-Quadcopter / \n",
"5-Dance / \n",
"3-Concert / 16-Performance art / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 235-Lipstick / 291-Rouge / \n",
"98-Festival / \n",
"14-Guitar / 20-Trailer / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 4612-Sepang International Circuit / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 301-Meal / 454-Sauce / 1567-Cook / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 81-Athlete / \n",
"35-Minecraft / 141-Microsoft Windows / \n",
"3-Concert / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 35-Minecraft / 394-Map / 2716-The Hunger Games / \n",
"25-Toy / 200-Doll / 439-Barbie / \n",
"3-Concert / 95-Talent show / \n",
"2-Video game / 55-Video game console / 199-Wii / 443-Wii U / \n",
"3-Concert / \n",
"5-Dance / 20-Trailer / \n",
"85-Combat / 117-Boxing / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 39-Drums / \n",
"0-Games / 2-Video game / 55-Video game console / \n",
"0-Games / 2-Video game / 19-PC game / 159-World of Warcraft / \n",
"0-Games / 54-Highlight film / 60-Basketball / 255-Slam dunk / \n",
"9-Music video / \n",
"28-Fashion / 1534-H&M / \n",
"7-Musician / 219-Accordion / 1994-Garmon / \n",
"261-Tool / 673-Saw / 2437-String trimmer / \n",
"121-Photography / \n",
"3-Concert / 68-Lighting / \n",
"261-Tool / \n",
"9-Music video / \n",
"3-Concert / 68-Lighting / \n",
"0-Games / 1187-Ragnarok Online / \n",
"8-Football / \n",
"0-Games / 2966-Patchwork / \n",
"1-Vehicle / 4-Car / 70-Driving / 94-Dashcam / 277-Sport utility vehicle / 332-Mercedes-Benz / 344-Coupé / 891-Mercedes-AMG / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / \n",
"1-Vehicle / 4-Car / 23-Mobile phone / 72-iPhone / \n",
"4665-Day spa / \n",
"3-Concert / 7-Musician / 21-Stadium / \n",
"609-Tattoo / 863-Body piercing / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 189-iPad / 272-iPod / 331-iPod touch / 554-iPhone 4 / 675-iPhone 4S / 1009-iPhone 3G / 1155-iPhone 3GS / 1504-iPad 3 / \n",
"602-Central processing unit / 3391-Alesis / \n",
"1-Vehicle / 10-Animal / 25-Toy / 74-Truck / 1420-Gear / 1611-Garbage truck / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 184-Vegetable / 500-Rice / 727-Curry / 2000-Mango / \n",
"9-Music video / \n",
"416-Apartment / \n",
"1-Vehicle / 4-Car / 245-BMW / 344-Coupé / \n",
"1572-Husband / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / \n",
"0-Games / 1167-Tanki Online / \n",
"0-Games / 2-Video game / 354-Egg / 426-Resident Evil / \n",
"20-Trailer / \n",
"3-Concert / \n",
"37-Gadget / 2009-Electromagnetic coil / \n",
"6-Animation / 2769-Decal / \n",
"0-Games / 10-Animal / 159-World of Warcraft / 171-Warcraft / 3301-Mop / \n",
"1789-Slender: The Eight Pages / \n",
"10-Animal / 80-Horse / \n",
"0-Games / 60-Basketball / 168-Basketball moves / 255-Slam dunk / \n",
"9-Music video / 10-Animal / 48-Pet / 71-Dog / \n",
"0-Games / 2-Video game / 9-Music video / 43-Call of Duty / 138-Call of Duty: Black Ops II / \n",
"1-Vehicle / 4-Car / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / \n",
"0-Games / 2-Video game / 23-Mobile phone / 29-Smartphone / 72-iPhone / 508-iPhone 5 / 760-iPhone 5s / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 42-Fishing / 91-Fish / 120-Winter / 147-River / 423-Coast / 825-Stream / 895-Trout / 1018-Salmon / 2094-Rainbow trout / \n",
"243-Sonic the Hedgehog / \n",
"3-Concert / 5-Dance / 7-Musician / \n",
"6-Animation / 15-Cartoon / \n",
"2337-Passport / \n",
"67-Cycling / 69-Bicycle / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 388-Fighter aircraft / \n",
"45-Cosmetics / 238-Nail / 300-Nail art / 304-Nail polish / 338-Manicure / \n",
"0-Games / 2-Video game / 2572-Lupin III / \n",
"18-Outdoor recreation / 22-Nature / 283-Rock / 347-Climbing / 828-Rock climbing / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / \n",
"0-Games / 712-Outer space / 3585-Space Engineers / \n",
"75-Wrestling / \n",
"6-Animation / 89-Comics / 269-Comic book / 803-DC Comics / 1106-Character / \n",
"12-Food / 52-Dish / 162-Medicine / 215-Eating / 780-Fat / \n",
"0-Games / 2-Video game / 6-Animation / 241-Fighting game / 311-Street Fighter / 461-Street Fighter IV / 496-Super Street Fighter IV / 1331-Ken Masters / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 154-Wheel / 158-Tire / \n",
"49-School / 191-Cheerleading / \n",
"3-Concert / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 297-Grand Theft Auto IV / 658-Grand Theft Auto: The Lost and Damned / \n",
"0-Games / 6-Animation / 15-Cartoon / 1118-Kirby / 1133-Kirby / 3421-Meta Knight / 4449-King Dedede / \n",
"0-Games / 8-Football / 65-Kick / 76-Ball / 595-Champion / \n",
"5-Dance / \n",
"0-Games / 18-Outdoor recreation / 155-Camera / \n",
"1-Vehicle / 259-Fire / 371-Firefighter / 796-Fire engine / \n",
"0-Games / 2-Video game / 464-First-person Shooter / \n",
"0-Games / 8-Football / 65-Kick / 73-Ball / \n",
"28-Fashion / 45-Cosmetics / \n",
"31-Disc jockey / \n",
"18-Outdoor recreation / 42-Fishing / 147-River / 225-Lake / 896-Largemouth bass / \n",
"122-Weight training / 204-Gym / 492-Floor / 529-Squat / 532-Barbell / 1389-Dumbbell / \n",
"95-Talent show / \n",
"0-Games / 2-Video game / 33-Weapon / \n",
"25-Toy / 396-One Piece / 435-Action figure / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 552-Honda Civic / \n",
"0-Games / 2-Video game / 15-Cartoon / 89-Comics / 185-Dragon Ball / 244-Goku / 418-Gohan / 876-Freeza / 1244-Trunks / 2384-Dragon Ball: Raging Blast / \n",
"0-Games / 1427-TalesRunner / \n",
"273-News program / \n",
"1-Vehicle / 4-Car / 74-Truck / 114-Engine / 527-Battery / 659-Van / \n",
"1-Vehicle / 4-Car / 360-Jeep / 1205-Jeep Wrangler / \n",
"0-Games / 1-Vehicle / 2-Video game / 11-Motorsport / 55-Video game console / 199-Wii / 456-Go-kart / 518-Mario Kart / 2750-Mii / \n",
"0-Games / 2-Video game / 35-Minecraft / 174-Farm / 1158-Wheat / \n",
"303-Paint / 319-Robot / \n",
"20-Trailer / \n",
"0-Games / 779-Point Blank / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / \n",
"9-Music video / \n",
"12-Food / 26-Cooking / 32-Recipe / 109-Cooking show / 1765-Muffin / \n",
"8-Football / \n",
"8-Football / 661-Stretching / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 120-Winter / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / 422-Call of Duty: Ghosts / \n",
"1-Vehicle / 4-Car / 41-Sports car / \n",
"0-Games / 2-Video game / 34-Action-adventure game / \n",
"56-Hair / 106-Hairstyle / 340-Afro-textured hair / \n",
"18-Outdoor recreation / 42-Fishing / 222-Surfing / \n",
"296-Saxophone / \n",
"242-Paper / 313-Book / \n",
"123-Naruto / \n",
"337-Advertising / \n",
"5-Dance / \n",
"1-Vehicle / 4-Car / 40-Road / \n",
"3-Concert / 7-Musician / 9-Music video / \n",
"10-Animal / 91-Fish / 258-Aquarium / \n",
"12-Food / 786-Juice / \n",
"6-Animation / 1579-Circle / 2010-Chart / \n",
"3-Concert / \n",
"8-Football / \n",
"0-Games / 35-Minecraft / 47-Personal computer / 131-Computer / 141-Microsoft Windows / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 207-Exhaust system / 790-Nissan GT-R / 1301-Nissan Skyline / 2247-Nissan Skyline GT-R / \n",
"141-Microsoft Windows / \n",
"3895-Curling / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 81-Athlete / \n",
"0-Games / 2-Video game / 241-Fighting game / 311-Street Fighter / 486-M.U.G.E.N / 764-The King of Fighters / \n",
"3-Concert / 7-Musician / 39-Drums / 98-Festival / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 125-Television / 487-Samsung Electronics / 1595-Television set / \n",
"204-Gym / 4667-Gold's Gym / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 428-Rail transport modelling / \n",
"102-Building / 170-Home improvement / 433-Construction / 1821-Office / \n",
"1-Vehicle / 11-Motorsport / 18-Outdoor recreation / 22-Nature / 27-Motorcycle / 105-Motorcycling / 213-Forest / 491-Enduro / \n",
"0-Games / 530-Cue sports / 568-Pool / 1334-Eight-ball / \n",
"36-Piano / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / \n",
"12-Food / 26-Cooking / 52-Dish / 109-Cooking show / 215-Eating / 274-Meat / 301-Meal / \n",
"18-Outdoor recreation / 22-Nature / 164-Beach / 233-Ocean / 423-Coast / 1453-Balloon / \n",
"79-American football / \n",
"28-Fashion / 45-Cosmetics / \n",
"6-Animation / 9-Music video / 1632-Villain / \n",
"68-Lighting / 99-Christmas / 729-Christmas decoration / 1098-Christmas lights / \n",
"0-Games / 2-Video game / 6-Animation / 586-Nintendo Entertainment System / \n",
"376-Flute / 1326-Recorder / \n",
"3-Concert / 5-Dance / \n",
"110-Album / \n",
"2-Video game / 43-Call of Duty / 431-Call of Duty: Advanced Warfare / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 240-Arcade game / 1333-Wangan Midnight / \n",
"10-Animal / 17-Racing / 80-Horse / 384-Horse racing / \n",
"1-Vehicle / 4-Car / 22-Nature / 114-Engine / \n",
"3-Concert / 7-Musician / \n",
"6-Animation / 3162-Postage stamp / 3544-Rubber stamp / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / \n",
"25-Toy / 139-Airport / 140-LEGO / 1079-Lego City / \n",
"1-Vehicle / 11-Motorsport / 124-Tractor / 156-Agriculture / 174-Farm / 252-Tractor pulling / 459-Plough / 660-Rural area / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 35-Minecraft / \n",
"25-Toy / 269-Comic book / \n",
"28-Fashion / 153-Dress / \n",
"164-Beach / 248-Hotel / 369-Resort / \n",
"1-Vehicle / 74-Truck / 208-Wood / 492-Floor / 619-Manufacturing / \n",
"6-Animation / 851-Digimon / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 231-Supercar / 344-Coupé / 349-Chevrolet / 406-Chevrolet / 761-Chevrolet Camaro / 1752-Chevrolet SS / \n",
"325-Vampire / \n",
"120-Winter / 657-Autumn / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / 318-Church / \n",
"48-Pet / 182-Puppy / 1278-Labrador Retriever / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 218-Eyelash / \n",
"10-Animal / 48-Pet / 132-Bird / \n",
"150-Violin / 1747-Erhu / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 70-Driving / 352-Need for Speed / 1056-Porsche 911 / 1600-Need for Speed: World / \n",
"0-Games / 33-Weapon / 34-Action-adventure game / 118-Grand Theft Auto V / 201-Grand Theft Auto: San Andreas / 297-Grand Theft Auto IV / \n",
"12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / 226-Dessert / 295-Chocolate / 684-Ice cream / 1322-Biscuit / 2418-Gelatin / 2777-Mousse / 2825-Rum / 2858-Gelatin dessert / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"45-Cosmetics / 238-Nail / 300-Nail art / 304-Nail polish / 307-Hand / 324-Finger / 338-Manicure / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 105-Motorcycling / \n",
"7-Musician / 9-Music video / 14-Guitar / 30-Drummer / \n",
"0-Games / 2-Video game / 43-Call of Duty / 134-Call of Duty: Black Ops / \n",
"1-Vehicle / 18-Outdoor recreation / 22-Nature / 67-Cycling / 69-Bicycle / 220-Mountain bike / 283-Rock / 284-Mountain biking / 347-Climbing / 800-Mountain pass / \n",
"1726-Harlem Shake (meme) / \n",
"3-Concert / 5-Dance / 16-Performance art / 357-DVD / \n",
"0-Games / 8-Football / \n",
"59-Winter sport / 84-Snow / 281-Snowboarding / 501-Snowboard / 827-Pump / \n",
"0-Games / 49-School / 60-Basketball / 95-Talent show / 191-Cheerleading / \n",
"131-Computer / 141-Microsoft Windows / \n",
"9-Music video / \n",
"1701-French fries / \n",
"33-Weapon / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"0-Games / 8-Football / 90-Sports game / 180-FIFA 15 / \n",
"61-Art / 1846-Mural / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / 265-Electronic keyboard / \n",
"1-Vehicle / 4-Car / 18-Outdoor recreation / \n",
"0-Games / 2-Video game / 352-Need for Speed / 875-Need for Speed: Most Wanted / \n",
"2-Video game / 240-Arcade game / 488-Door / 731-Coin / \n",
"0-Games / 35-Minecraft / \n",
"462-Table / 695-Balloon / 1327-Minnie Mouse / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 169-Rallying / 245-BMW / 522-Helmet / \n",
"661-Stretching / \n",
"159-World of Warcraft / 171-Warcraft / 372-Playing card / 791-Magic: The Gathering / 1021-Booster pack / 1049-Collectible card game / \n",
"0-Games / 2-Video game / 20-Trailer / 34-Action-adventure game / 89-Comics / 294-Batman / 838-Batman: Arkham / 2084-Batman: Arkham Origins / \n",
"0-Games / 2-Video game / 55-Video game console / 111-PlayStation 3 / 317-PlayStation / 1482-PlayStation Network / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 135-Drifting / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 178-Samsung Galaxy / 2508-Samsung Galaxy Ace / \n",
"0-Games / 463-Cricket / \n",
"3-Concert / 5-Dance / \n",
"12-Food / 215-Eating / 968-Pie / 1594-Nerd / 3613-Apple pie / \n",
"861-Mixing console / 3391-Alesis / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"3-Concert / 7-Musician / \n",
"25-Toy / 140-LEGO / \n",
"12-Food / 1227-Sushi / \n",
"85-Combat / 117-Boxing / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 161-Train station / 224-Rapid transit / \n",
"9-Music video / 98-Festival / \n",
"1-Vehicle / 4-Car / 299-Volkswagen Passenger Cars / \n",
"1-Vehicle / 4-Car / 519-Recreational vehicle / 742-Countertop / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 114-Engine / 154-Wheel / 207-Exhaust system / \n",
"9-Music video / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / 1936-Bouzouki / \n",
"0-Games / 596-Monster Hunter / 1339-Monster Hunter Freedom Unite / 2994-Monster Hunter Freedom / \n",
"3-Concert / 1303-Fullmetal Alchemist / \n",
"10-Animal / 22-Nature / 1039-Tiger / \n",
"8-Football / \n",
"320-Super Smash Bros. / 398-The Legend of Zelda / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / 40-Road / 105-Motorcycling / 212-Highway / 608-Wheelie / \n",
"10-Animal / 709-Bull / \n",
"1-Vehicle / 27-Motorcycle / 105-Motorcycling / 1395-Valve / \n",
"36-Piano / \n",
"12-Food / 26-Cooking / 32-Recipe / 226-Dessert / 236-Baking / 295-Chocolate / 1223-Pastry / 1576-Doughnut / \n",
"0-Games / 2-Video game / \n",
"1-Vehicle / 4-Car / \n",
"13-Musical ensemble / \n",
"0-Games / 8-Football / 65-Kick / \n",
"31-Disc jockey / \n",
"6-Animation / 10-Animal / 15-Cartoon / \n",
"22-Nature / 279-Garden / 369-Resort / \n",
"1-Vehicle / 4-Car / 303-Paint / \n",
"86-Plant / 136-Gardening / 279-Garden / \n",
"66-Bollywood / \n",
"6-Animation / 183-Manga / \n",
"121-Photography / 155-Camera / 566-Digital camera / 1117-Camcorder / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"133-Nightclub / \n",
"162-Medicine / 925-Brain / \n",
"0-Games / 2-Video game / 116-Pokémon / 186-Pokémon / 4521-Deoxys / \n",
"56-Hair / 106-Hairstyle / \n",
"59-Winter sport / 112-Ice skating / 262-Ice / 316-Figure skating / 632-Ice dancing / 833-Ice skate / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 207-Exhaust system / \n",
"0-Games / 51-Strategy video game / 373-Clash of Clans / \n",
"1113-Toilet / 1308-Shower / 2472-Tap / \n",
"3-Concert / 726-Clarinet / \n",
"390-Model / \n",
"3-Concert / 1231-Mandolin / \n",
"162-Medicine / 196-Eye / \n",
"0-Games / 2-Video game / 23-Mobile phone / 29-Smartphone / 72-iPhone / 113-House / 1234-Mouse / \n",
"1-Vehicle / 1409-Missile / \n",
"227-Water / \n",
"2794-Marionette / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 94-Dashcam / 490-Rain / \n",
"20-Trailer / \n",
"0-Games / 67-Cycling / 69-Bicycle / 570-Road bicycle racing / \n",
"6-Animation / 15-Cartoon / 272-iPod / 1674-Ouran High School Host Club / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"1-Vehicle / 4-Car / 74-Truck / 114-Engine / 349-Chevrolet / 375-Pickup truck / 406-Chevrolet / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 40-Road / 94-Dashcam / 105-Motorcycling / \n",
"0-Games / 256-Tennis / \n",
"0-Games / 19-PC game / 35-Minecraft / \n",
"56-Hair / 60-Basketball / \n",
"36-Piano / 53-Keyboard / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 84-Snow / 120-Winter / 135-Drifting / 680-Subaru / 1041-Subaru Impreza / \n",
"12-Food / 26-Cooking / 32-Recipe / 269-Comic book / 367-Drink / 613-Bar / 904-Cocktail / 1669-Vodka / \n",
"1487-Reborn doll / \n",
"28-Fashion / 206-Shoe / \n",
"10-Animal / 588-Reptile / 1444-Lizard / 3060-Chameleon / \n",
"18-Outdoor recreation / 42-Fishing / 222-Surfing / 2415-Big wave surfing / \n",
"99-Christmas / 558-Santa Claus / \n",
"5-Dance / 115-Ballet / \n",
"0-Games / 2-Video game / 116-Pokémon / 186-Pokémon / 4267-GameShark / \n",
"83-Skateboarding / 108-Skateboard / 221-Skatepark / 2919-Skate 2 / \n",
"237-Final Fantasy / 1131-Final Fantasy XIII / 2506-Final Fantasy XIII-2 / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / \n",
"1-Vehicle / 10-Animal / 18-Outdoor recreation / 42-Fishing / 87-Boat / 91-Fish / 233-Ocean / 268-Recreational fishing / 698-Fisherman / 3763-Atlantic bluefin tuna / \n",
"379-Touhou Project / \n",
"2-Video game / 3515-Gas tungsten arc welding / 3929-Gas metal arc welding / \n",
"0-Games / 35-Minecraft / \n",
"5-Dance / 16-Performance art / 46-Choir / \n",
"283-Rock / 492-Floor / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / 78-Wedding / 190-Bride / \n",
"264-Star Wars / 714-Jedi / 717-Darth Vader / \n",
"5-Dance / 16-Performance art / 130-Gymnastics / 1255-Ribbon / \n",
"22-Nature / \n",
"7-Musician / 36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / 307-Hand / \n",
"8-Football / \n",
"66-Bollywood / \n",
"31-Disc jockey / 1143-Marimba / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 84-Snow / 105-Motorcycling / 120-Winter / \n",
"0-Games / 2-Video game / 3717-Suikoden / \n",
"1-Vehicle / 164-Beach / 423-Coast / \n",
"88-Machine / 400-Woodturning / 972-Lathe / \n",
"6-Animation / 15-Cartoon / 99-Christmas / \n",
"1-Vehicle / 88-Machine / 249-Heavy equipment / 817-Bulldozer / \n",
"86-Plant / 136-Gardening / 279-Garden / 1460-Herb / \n",
"1-Vehicle / 136-Gardening / 249-Heavy equipment / 363-Soil / 2138-Compost / \n",
"3-Concert / 7-Musician / 14-Guitar / 24-String instrument / 30-Drummer / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 172-Motocross / \n",
"3-Concert / \n",
"10-Animal / 48-Pet / 71-Dog / 84-Snow / 280-Wildlife / \n",
"0-Games / 2-Video game / 19-PC game / 229-Halo / 470-Halo 3 / \n",
"0-Games / 1427-TalesRunner / \n",
"10-Animal / 137-Tree / 542-Potato / 725-Human / \n",
"12-Food / 2496-Kebab / \n",
"0-Games / 2-Video game / 51-Strategy video game / 537-Zoo / 865-Turtle / 4255-Zoo Tycoon 2 / \n",
"117-Boxing / \n",
"46-Choir / \n",
"3-Concert / 7-Musician / 24-String instrument / \n",
"3-Concert / 5-Dance / 16-Performance art / 95-Talent show / 115-Ballet / 130-Gymnastics / \n",
"10-Animal / 22-Nature / 280-Wildlife / 540-Insect / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 135-Drifting / 2739-Midnight Club: Los Angeles / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 2-Video game / 43-Call of Duty / 216-Call of Duty: Modern Warfare 3 / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / \n",
"0-Games / 13-Musical ensemble / 203-Marching band / \n",
"9-Music video / 66-Bollywood / \n",
"1-Vehicle / 4-Car / 50-Aircraft / 259-Fire / 559-Night / \n",
"61-Art / 97-Drawing / 479-Graffiti / 1385-Street art / 1846-Mural / \n",
"0-Games / 60-Basketball / 1399-Streetball / \n",
"45-Cosmetics / 56-Hair / \n",
"283-Rock / \n",
"0-Games / 88-Machine / 240-Arcade game / 535-Slot machine / 3235-Monopoly / \n",
"5-Dance / \n",
"731-Coin / 912-Silver / \n",
"6-Animation / 15-Cartoon / 505-Mega Man / 1664-Zero / \n",
"0-Games / 2-Video game / 88-Machine / 535-Slot machine / \n",
"8-Football / \n",
"1-Vehicle / 25-Toy / 62-Train / \n",
"10-Animal / 48-Pet / 71-Dog / \n",
"0-Games / \n",
"5-Dance / \n",
"8-Football / \n",
"12-Food / 26-Cooking / 210-Cookware and bakeware / 589-Grilling / 620-Barbecue / 1085-Barbecue grill / \n",
"10-Animal / 22-Nature / 440-Lion / 537-Zoo / \n",
"3-Concert / \n",
"8-Football / \n",
"72-iPhone / 272-iPod / \n",
"85-Combat / \n",
"6-Animation / 35-Minecraft / 99-Christmas / \n",
"1-Vehicle / 27-Motorcycle / 1380-Kawasaki motorcycles / 1951-Sport bike / 2404-Kawasaki Ninja / 2494-Ninja ZX-6R / 3873-Kawasaki Ninja 300 / \n",
"6-Animation / 15-Cartoon / 396-One Piece / \n",
"3-Concert / 7-Musician / 9-Music video / \n",
"123-Naruto / \n",
"3-Concert / 7-Musician / \n",
"1-Vehicle / 4-Car / 84-Snow / 120-Winter / 277-Sport utility vehicle / 680-Subaru / 1286-Subaru / 2862-Subaru Forester / \n",
"59-Winter sport / 84-Snow / 177-Skiing / 281-Snowboarding / 3694-Burton Snowboards / \n",
"7-Musician / 13-Musical ensemble / 14-Guitar / \n",
"0-Games / 2-Video game / 474-Destiny / \n",
"0-Games / 51-Strategy video game / 165-League of Legends / \n",
"7-Musician / 14-Guitar / \n",
"10-Animal / 48-Pet / 166-Cat / 346-Kitten / \n",
"99-Christmas / 729-Christmas decoration / \n",
"1-Vehicle / 64-Transport / 286-Bus / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 123-Naruto / \n",
"5-Dance / \n",
"979-Star Trek / 2981-Spock / \n",
"5-Dance / 16-Performance art / 115-Ballet / 980-Solo dance / 3643-Starbound / \n",
"0-Games / 2-Video game / 23-Mobile phone / 29-Smartphone / 72-iPhone / 1009-iPhone 3G / \n",
"3-Concert / \n",
"237-Final Fantasy / 1371-Yuna / 1706-Final Fantasy X-2 / \n",
"2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 178-Samsung Galaxy / 899-Samsung Galaxy S III / \n",
"3-Concert / \n",
"372-Playing card / 746-Card manipulation / \n",
"6-Animation / 10-Animal / 15-Cartoon / 577-Penguin / \n",
"1061-Marriage proposal / \n",
"6-Animation / 15-Cartoon / 750-Sailor Moon / 1342-Sailor Moon / \n",
"3-Concert / \n",
"8-Football / \n",
"242-Paper / 370-Origami / 1023-Rose / 1403-Flower bouquet / \n",
"0-Games / 2-Video game / 269-Comic book / 489-Board game / 494-Card game / \n",
"1-Vehicle / 59-Winter sport / 84-Snow / 120-Winter / 351-Jumping / 608-Wheelie / 610-Snowmobile / 1201-Ski-Doo / 1210-Sled / \n",
"1-Vehicle / 25-Toy / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 507-Thomas the Tank Engine / \n",
"9-Music video / \n",
"2-Video game / 192-Battlefield / 1370-Battlefield 2 / \n",
"110-Album / 956-Glasses / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / \n",
"1-Vehicle / 2143-18 Wheels of Steel / \n",
"83-Skateboarding / 108-Skateboard / \n",
"5-Dance / \n",
"1-Vehicle / 27-Motorcycle / \n",
"313-Book / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 81-Athlete / \n",
"55-Video game console / \n",
"1-Vehicle / 4-Car / 92-Radio-controlled model / 202-Radio-controlled car / 945-MINI Cooper / \n",
"14-Guitar / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / 70-Driving / \n",
"49-School / 860-Christian Church / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"85-Combat / 699-Kick / 1146-Wing Chun / \n",
"0-Games / 311-Street Fighter / 461-Street Fighter IV / 496-Super Street Fighter IV / 563-Ryu / \n",
"3969-Patience / \n",
"0-Games / 2-Video game / 387-Spider-Man / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 154-Wheel / 299-Volkswagen Passenger Cars / 493-Rim / 526-Volkswagen / 531-Hatchback / 668-Volkswagen Golf / \n",
"12-Food / \n",
"3-Concert / \n",
"6-Animation / 10-Animal / 15-Cartoon / 61-Art / 97-Drawing / 132-Bird / 148-Painting / 408-Sketch / \n",
"13-Musical ensemble / 30-Drummer / 39-Drums / 44-Drums / 49-School / 128-Cymbal / 146-Snare drum / 203-Marching band / \n",
"85-Combat / 117-Boxing / \n",
"1-Vehicle / 42-Fishing / 87-Boat / 88-Machine / 896-Largemouth bass / \n",
"1-Vehicle / 4-Car / 41-Sports car / 154-Wheel / 3834-Porsche Panamera / \n",
"3-Concert / \n",
"28-Fashion / \n",
"2-Video game / 25-Toy / 140-LEGO / 434-Ninja / 884-Lego minifigure / 951-Teenage Mutant Ninja Turtles / \n",
"0-Games / 2-Video game / 211-RuneScape / \n",
"83-Skateboarding / 108-Skateboard / \n",
"1-Vehicle / 18-Outdoor recreation / 42-Fishing / 87-Boat / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 1187-Ragnarok Online / \n",
"0-Games / 2-Video game / 6-Animation / 89-Comics / 185-Dragon Ball / 244-Goku / \n",
"1436-Draco Malfoy / \n",
"9-Music video / \n",
"261-Tool / 1033-Drill / 1226-Metalworking / \n",
"3-Concert / \n",
"630-Superman / 2051-Justice League / \n",
"9-Music video / \n",
"5-Dance / 164-Beach / \n",
"66-Bollywood / \n",
"2-Video game / 43-Call of Duty / 216-Call of Duty: Modern Warfare 3 / \n",
"122-Weight training / 204-Gym / 2505-Slingshot / \n",
"9-Music video / 1183-Karaoke box / \n",
"3-Concert / 68-Lighting / 133-Nightclub / \n",
"85-Combat / \n",
"1-Vehicle / 4-Car / 245-BMW / 3320-BMW 5 Series (E60) / \n",
"1-Vehicle / 4-Car / \n",
"775-Bee / 4198-Khene / \n",
"20-Trailer / \n",
"2-Video game / 55-Video game console / 127-Xbox 360 / 181-Xbox / 386-Xbox / 700-Hard disk drive / \n",
"99-Christmas / 137-Tree / 964-Christmas tree / \n",
"96-Soldier / 99-Christmas / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / \n",
"28-Fashion / \n",
"93-Comedy / \n",
"51-Strategy video game / 366-Dragon / 1718-Spore / \n",
"102-Building / 248-Hotel / 574-Elevator / 1746-Otis Elevator Company / 3809-Marriott International / \n",
"0-Games / 788-Juggling / \n",
"20-Trailer / 625-Factory / \n",
"9-Music video / 133-Nightclub / \n",
"0-Games / 8-Football / 54-Highlight film / \n",
"205-Newscaster / 624-Number / \n",
"313-Book / \n",
"0-Games / 1-Vehicle / 11-Motorsport / 76-Ball / 169-Rallying / 256-Tennis / 974-Racket / 1337-Serve / \n",
"0-Games / 6-Animation / 15-Cartoon / \n",
"208-Wood / 956-Glasses / \n",
"75-Wrestling / 1042-Jumbotron / \n",
"1-Vehicle / 4-Car / 6-Animation / \n",
"23-Mobile phone / 37-Gadget / 47-Personal computer / 145-Tablet computer / 189-iPad / 1504-iPad 3 / 2253-CNET / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 135-Drifting / 3409-Nissan Altima / \n",
"8-Football / 205-Newscaster / \n",
"89-Comics / \n",
"130-Gymnastics / \n",
"9-Music video / \n",
"28-Fashion / 121-Photography / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 585-Lawn / 741-Mower / 805-Lawn mower / \n",
"3-Concert / \n",
"85-Combat / 117-Boxing / \n",
"0-Games / 2-Video game / 19-PC game / 420-The Elder Scrolls V: Skyrim / 441-The Elder Scrolls / \n",
"31-Disc jockey / \n",
"6-Animation / 625-Factory / 2189-Warehouse / \n",
"66-Bollywood / \n",
"0-Games / 2-Video game / 43-Call of Duty / 431-Call of Duty: Advanced Warfare / 737-Joker / \n",
"36-Piano / 885-Bell / \n",
"49-School / 747-Retail / \n",
"0-Games / 3387-Monster Retsuden Oreca Battle / \n",
"12-Food / 2676-Citrus / \n",
"18-Outdoor recreation / 22-Nature / 147-River / 283-Rock / 614-Hiking / 722-Waterfall / 825-Stream / \n",
"14-Guitar / 24-String instrument / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 124-Tractor / 805-Lawn mower / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 3345-V8 Supercars / \n",
"5-Dance / \n",
"144-Amusement park / 321-Roller coaster / \n",
"196-Eye / \n",
"1-Vehicle / 4-Car / 25-Toy / 74-Truck / 140-LEGO / \n",
"3471-Armlock / \n",
"86-Plant / 136-Gardening / 279-Garden / \n",
"3-Concert / 5-Dance / 13-Musical ensemble / 16-Performance art / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 38-Orchestra / \n",
"0-Games / 163-Running / \n",
"36-Piano / 53-Keyboard / 446-Sheet music / 592-Synthesia / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 47-Personal computer / 145-Tablet computer / 178-Samsung Galaxy / 899-Samsung Galaxy S III / 1727-Samsung Galaxy Note 3 / \n",
"117-Boxing / \n",
"3-Concert / 68-Lighting / \n",
"5-Dance / 16-Performance art / 95-Talent show / 430-Ballroom dance / \n",
"9-Music video / \n",
"2-Video game / 1251-The Witcher / 1401-The Witcher 3: Wild Hunt / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"3-Concert / 5-Dance / 7-Musician / 16-Performance art / 361-Flamenco / \n",
"3-Concert / 7-Musician / \n",
"61-Art / 97-Drawing / \n",
"102-Building / 113-House / 840-Museum / \n",
"102-Building / 339-Architecture / \n",
"0-Games / 2-Video game / 51-Strategy video game / \n",
"0-Games / 2-Video game / 123-Naruto / 1410-The Elder Scrolls IV: Oblivion / \n",
"10-Animal / 280-Wildlife / 440-Lion / 3986-Cougar / \n",
"257-Counter-Strike / 1078-Installation art / \n",
"322-Mixtape / \n",
"86-Plant / 136-Gardening / 279-Garden / 1176-Pumpkin / \n",
"3-Concert / 7-Musician / 24-String instrument / \n",
"66-Bollywood / \n",
"3-Concert / 30-Drummer / 68-Lighting / 2032-Pepsi / \n",
"5-Dance / 66-Bollywood / 293-Slide show / \n",
"12-Food / 86-Plant / 2343-Caridean Shrimp / \n",
"3-Concert / 16-Performance art / \n",
"1-Vehicle / 59-Winter sport / 112-Ice skating / 142-Hockey / \n",
"0-Games / 59-Winter sport / 112-Ice skating / 142-Hockey / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 57-Race track / 70-Driving / 135-Drifting / \n",
"5-Dance / 9-Music video / 31-Disc jockey / \n",
"1-Vehicle / 4-Car / 11-Motorsport / \n",
"6-Animation / 15-Cartoon / 2944-La Tale / \n",
"0-Games / 59-Winter sport / 81-Athlete / 112-Ice skating / 142-Hockey / 312-Ice rink / \n",
"2-Video game / 23-Mobile phone / 37-Gadget / 47-Personal computer / 145-Tablet computer / 189-iPad / 1926-Amazon Kindle / 2908-Kindle Fire / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / \n",
"1-Vehicle / 4-Car / 129-Aviation / 695-Balloon / 1879-Hot air balloon / \n",
"12-Food / 26-Cooking / 32-Recipe / 236-Baking / \n",
"25-Toy / 425-City / 1850-Playmobil / \n",
"122-Weight training / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"31-Disc jockey / \n",
"102-Building / 170-Home improvement / 208-Wood / 2221-Deck / \n",
"61-Art / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 824-Terrier / \n",
"13-Musical ensemble / 30-Drummer / 39-Drums / \n",
"6-Animation / 15-Cartoon / 99-Christmas / 2047-Popeye / \n",
"3-Concert / \n",
"75-Wrestling / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 378-Drag racing / 4223-Funny Car / \n",
"2-Video game / 37-Gadget / 55-Video game console / 111-PlayStation 3 / 176-PlayStation 4 / 292-Handheld game console / 317-PlayStation / 682-PlayStation Vita / 1482-PlayStation Network / \n",
"12-Food / 32-Recipe / \n",
"31-Disc jockey / \n",
"0-Games / 10-Animal / 48-Pet / \n",
"61-Art / 479-Graffiti / 1385-Street art / \n",
"342-Windows Media Video / \n",
"10-Animal / 42-Fishing / 144-Amusement park / 1075-Whale / 1135-Dolphin / 1780-Killer whale / \n",
"0-Games / 2-Video game / 25-Toy / 140-LEGO / 2276-Lego Marvel Super Heroes / \n",
"6-Animation / 25-Toy / \n",
"3-Concert / 98-Festival / \n",
"8-Football / \n",
"6-Animation / 20-Trailer / 1899-E-book / \n",
"18-Outdoor recreation / 213-Forest / 639-Fiddle / 1137-Crash Bandicoot / 2569-Crash Bandicoot / 3720-Crash Bandicoot 2: Cortex Strikes Back / \n",
"1483-United States Air Force / \n",
"0-Games / 76-Ball / 772-Bowling / 930-Ten-pin bowling / 1217-Bowling ball / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 40-Road / 41-Sports car / 118-Grand Theft Auto V / 231-Supercar / 1979-Custom car / \n",
"12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / 109-Cooking show / 179-Kitchen / 215-Eating / 966-Sandwich / 1355-Vegetarian food / \n",
"1-Vehicle / 4-Car / 70-Driving / 270-Sedan / 680-Subaru / 1286-Subaru / 3040-Subaru Legacy / \n",
"988-Ibiza / \n",
"192-Battlefield / 374-Battlefield 3 / \n",
"6-Animation / 15-Cartoon / \n",
"10-Animal / 22-Nature / 86-Plant / 540-Insect / \n",
"0-Games / 60-Basketball / 81-Athlete / 168-Basketball moves / 255-Slam dunk / \n",
"0-Games / 75-Wrestling / 977-WWE '13 / \n",
"28-Fashion / 56-Hair / 106-Hairstyle / 583-Wig / 607-Lace / \n",
"8-Football / 21-Stadium / 595-Champion / \n",
"29-Smartphone / 504-Web page / \n",
"5-Dance / \n",
"1-Vehicle / 40-Road / 94-Dashcam / 663-Street / \n",
"2-Video game / 20-Trailer / 739-Guild Wars / \n",
"75-Wrestling / \n",
"5-Dance / 16-Performance art / 77-Arena / 253-Carnival / \n",
"9-Music video / 15-Cartoon / 290-Pony / 343-My Little Pony / \n",
"1-Vehicle / 62-Train / 64-Transport / 94-Dashcam / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 691-Cargo / 821-Rail freight transport / \n",
"12-Food / 26-Cooking / 58-Cuisine / 315-Knife / 1567-Cook / \n",
"1-Vehicle / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 515-Dodge / \n",
"0-Games / 1138-Tower of Saviors / \n",
"6-Animation / \n",
"9-Music video / \n",
"163-Running / \n",
"7-Musician / 13-Musical ensemble / 49-School / 79-American football / 203-Marching band / 250-Parade / \n",
"39-Drums / 44-Drums / \n",
"28-Fashion / 45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / \n",
"75-Wrestling / \n",
"13-Musical ensemble / 754-Gramophone record / \n",
"3-Concert / 5-Dance / 133-Nightclub / \n",
"1-Vehicle / 3311-Hovercraft / \n",
"18-Outdoor recreation / 22-Nature / 164-Beach / 233-Ocean / 423-Coast / 722-Waterfall / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 157-Jet aircraft / 388-Fighter aircraft / 758-Air force / 1483-United States Air Force / \n",
"3-Concert / \n",
"1791-Stargate / \n",
"12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / 109-Cooking show / 179-Kitchen / 236-Baking / 485-Dough / \n",
"7-Musician / 13-Musical ensemble / 230-Brass instrument / 296-Saxophone / 427-Trumpet / 634-Trombone / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / \n",
"5-Dance / \n",
"287-Human swimming / \n",
"10-Animal / 233-Ocean / 288-Underwater / 412-Monster / \n",
"0-Games / 2-Video game / 420-The Elder Scrolls V: Skyrim / 441-The Elder Scrolls / \n",
"1-Vehicle / 4-Car / \n",
"60-Basketball / \n",
"6-Animation / 15-Cartoon / \n",
"7-Musician / 14-Guitar / 1754-Rocksmith / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 216-Call of Duty: Modern Warfare 3 / \n",
"3756-Carpet cleaning / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 76-Ball / \n",
"1-Vehicle / 4-Car / 40-Road / 94-Dashcam / 212-Highway / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 388-Fighter aircraft / \n",
"5-Dance / \n",
"681-Leather / 1082-Knot / 2814-Keychain / \n",
"0-Games / 54-Highlight film / 60-Basketball / 4466-Korfball / \n",
"22-Nature / 137-Tree / 213-Forest / 444-Family / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / \n",
"12-Food / 232-Cake / 485-Dough / \n",
"10-Animal / 80-Horse / 247-Stallion / 289-Dressage / 342-Windows Media Video / \n",
"45-Cosmetics / \n",
"12-Food / 26-Cooking / \n",
"191-Cheerleading / \n",
"5-Dance / 16-Performance art / 25-Toy / 1094-Puppet / 1327-Minnie Mouse / \n",
"7-Musician / 14-Guitar / \n",
"1-Vehicle / 74-Truck / \n",
"0-Games / 76-Ball / 3355-Bingo / \n",
"0-Games / 6-Animation / 261-Tool / 1106-Character / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / \n",
"341-Sketch comedy / \n",
"78-Wedding / 190-Bride / \n",
"238-Nail / 300-Nail art / 338-Manicure / \n",
"3-Concert / \n",
"5-Dance / 199-Wii / 4437-Just Dance 2 / \n",
"67-Cycling / 69-Bicycle / 377-BMX bike / 520-Bicycle frame / \n",
"7-Musician / 46-Choir / 49-School / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"3-Concert / \n",
"28-Fashion / 45-Cosmetics / 56-Hair / 106-Hairstyle / 173-Eye shadow / 193-Eye liner / 291-Rouge / 340-Afro-textured hair / \n",
"1-Vehicle / 4-Car / 154-Wheel / 245-BMW / 3058-BMW 7 Series / \n",
"6-Animation / 15-Cartoon / 183-Manga / 2408-Lucky Star / \n",
"56-Hair / 106-Hairstyle / 954-Dreadlocks / \n",
"86-Plant / 106-Hairstyle / \n",
"1-Vehicle / 4-Car / 74-Truck / 158-Tire / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 380-Assassin's Creed / 1140-Assassin's Creed III / \n",
"293-Slide show / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 178-Samsung Galaxy / 487-Samsung Electronics / \n",
"1-Vehicle / 4-Car / \n",
"5-Dance / \n",
"6-Animation / 15-Cartoon / \n",
"7-Musician / 14-Guitar / 24-String instrument / \n",
"25-Toy / 89-Comics / 435-Action figure / 2121-Marvel Legends / \n",
"16-Performance art / \n",
"1-Vehicle / 27-Motorcycle / \n",
"1-Vehicle / 11-Motorsport / 124-Tractor / 156-Agriculture / 174-Farm / 252-Tractor pulling / 363-Soil / 459-Plough / \n",
"253-Carnival / \n",
"25-Toy / 140-LEGO / \n",
"1-Vehicle / 4-Car / 40-Road / 94-Dashcam / 207-Exhaust system / 854-Diesel engine / 947-Muffler / \n",
"5-Dance / 16-Performance art / 430-Ballroom dance / 889-Swing / \n",
"1-Vehicle / 4-Car / 197-Trail / 1895-Werewolf / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 189-iPad / 508-iPhone 5 / 760-iPhone 5s / \n",
"710-Handball / \n",
"1-Vehicle / 4-Car / \n",
"102-Building / 248-Hotel / \n",
"12-Food / 26-Cooking / 52-Dish / 58-Cuisine / 1116-Goat / \n",
"5-Dance / 16-Performance art / 332-Mercedes-Benz / \n",
"10-Animal / 588-Reptile / 2222-Crocodile / 3785-Crocodilia / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / \n",
"5-Dance / 59-Winter sport / 112-Ice skating / 316-Figure skating / \n",
"1-Vehicle / 4-Car / 114-Engine / 533-Oil / \n",
"10-Animal / 48-Pet / 71-Dog / 824-Terrier / \n",
"20-Trailer / \n",
"5-Dance / \n",
"153-Dress / 205-Newscaster / \n",
"1-Vehicle / 4-Car / 1134-Taxicab / \n",
"2-Video game / 23-Mobile phone / 29-Smartphone / 72-iPhone / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 172-Motocross / 278-GoPro / 522-Helmet / 1323-Helmet camera / \n",
"3-Concert / \n",
"223-Weather / 419-Earth / 448-Planet / 593-Moon / 654-Sun / 1918-Comet / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 1187-Ragnarok Online / \n",
"83-Skateboarding / 108-Skateboard / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 816-Gundam / 1343-Super Robot Wars / \n",
"33-Weapon / 1409-Missile / 4341-Anti-aircraft warfare / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 70-Driving / 231-Supercar / 344-Coupé / 2335-Aston Martin / 4711-Aston Martin V8 Vantage (2005) / \n",
"12-Food / 695-Balloon / 1972-Toy balloon / \n",
"0-Games / 2-Video game / 19-PC game / 464-First-person Shooter / 928-Borderlands 2 / 3434-Gearbox Software / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 40-Road / 57-Race track / 1867-Honda CB600F / 2494-Ninja ZX-6R / \n",
"23-Mobile phone / 29-Smartphone / 101-Telephone / 1776-Samsung Galaxy Note II / \n",
"28-Fashion / 305-Ski / \n",
"75-Wrestling / 85-Combat / \n",
"20-Trailer / 1014-Movieclips / \n",
"0-Games / 411-MapleStory / \n",
"31-Disc jockey / 567-Compact disc / \n",
"0-Games / 34-Action-adventure game / \n",
"3-Concert / 16-Performance art / \n",
"5-Dance / 16-Performance art / 1579-Circle / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 179-Kitchen / 454-Sauce / 589-Grilling / 620-Barbecue / 1018-Salmon / 1085-Barbecue grill / 1256-Smoking / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 169-Rallying / \n",
"1-Vehicle / 4-Car / 64-Transport / 74-Truck / 1429-Dump truck / 4242-Mack Trucks / \n",
"10-Animal / 48-Pet / 182-Puppy / 824-Terrier / \n",
"0-Games / 2-Video game / 19-PC game / 271-Tank / 850-Left 4 Dead / 963-Left 4 Dead 2 / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 1588-Prince of Persia / 1653-Prince / 3894-Prince of Persia: The Sands of Time / \n",
"0-Games / 2-Video game / 33-Weapon / 1749-Titanfall / \n",
"83-Skateboarding / 1105-Fingerboard / \n",
"164-Beach / 1208-Coconut / \n",
"9-Music video / \n",
"12-Food / 84-Snow / 262-Ice / \n",
"44-Drums / 3465-Davul / \n",
"10-Animal / 12-Food / 42-Fishing / 91-Fish / 893-Fish as food / 1018-Salmon / 2125-Fillet / \n",
"5-Dance / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 30-Drummer / 39-Drums / 44-Drums / 133-Nightclub / \n",
"444-Family / \n",
"269-Comic book / \n",
"13-Musical ensemble / \n",
"3804-Music of Eritrea / \n",
"0-Games / 2-Video game / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / 44-Drums / 146-Snare drum / \n",
"0-Games / 6-Animation / 116-Pokémon / 186-Pokémon / \n",
"0-Games / 2-Video game / 23-Mobile phone / 687-Taiko no Tatsujin / \n",
"22-Nature / \n",
"5-Dance / 3088-Mosquito / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 169-Rallying / \n",
"0-Games / 2-Video game / 19-PC game / 2491-Spec Ops: The Line / \n",
"0-Games / 2-Video game / 1503-Dungeon Fighter Online / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 239-Roasting / 771-Noodle / 1295-Stir frying / 3379-Prawn / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / \n",
"215-Eating / 770-Banana / \n",
"295-Chocolate / \n",
"1-Vehicle / 4-Car / \n",
"0-Games / 8-Football / 21-Stadium / 81-Athlete / \n",
"110-Album / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 216-Call of Duty: Modern Warfare 3 / \n",
"18-Outdoor recreation / 59-Winter sport / 84-Snow / 120-Winter / 177-Skiing / 305-Ski / \n",
"0-Games / 8-Football / 21-Stadium / 358-Rugby football / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / \n",
"6-Animation / 15-Cartoon / \n",
"6-Animation / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / \n",
"59-Winter sport / 120-Winter / 177-Skiing / 267-Mountain / 278-GoPro / 347-Climbing / \n",
"49-School / 444-Family / \n",
"31-Disc jockey / 3458-DMC World DJ Championships / \n",
"0-Games / 2-Video game / 34-Action-adventure game / \n",
"20-Trailer / \n",
"28-Fashion / 120-Winter / 785-Jeans / 2188-Coat / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"3-Concert / 7-Musician / 14-Guitar / 38-Orchestra / \n",
"1-Vehicle / 12-Food / 156-Agriculture / \n",
"3-Concert / 13-Musical ensemble / 253-Carnival / \n",
"36-Piano / 53-Keyboard / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / 114-Engine / 947-Muffler / \n",
"0-Games / 259-Fire / 1138-Tower of Saviors / \n",
"1-Vehicle / 12-Food / 124-Tractor / 156-Agriculture / 174-Farm / 931-Combine Harvester / 1158-Wheat / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 184-Vegetable / 301-Meal / 1707-Microwave oven / 2066-Omelette / \n",
"5-Dance / 31-Disc jockey / \n",
"10-Animal / 48-Pet / 166-Cat / 182-Puppy / 346-Kitten / \n",
"9-Music video / 110-Album / \n",
"78-Wedding / \n",
"0-Games / 2-Video game / 85-Combat / 1166-Dark Souls II / \n",
"3-Concert / 594-Music festival / \n",
"0-Games / 2-Video game / 35-Minecraft / 394-Map / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"0-Games / 19-PC game / 51-Strategy video game / 1967-Sangokushi Taisen / \n",
"1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 127-Xbox 360 / 135-Drifting / 169-Rallying / 1351-Rallycross / 3047-Colin McRae: Dirt 2 / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"66-Bollywood / \n",
"144-Amusement park / 369-Resort / 916-Spa / \n",
"2-Video game / 6-Animation / 15-Cartoon / \n",
"12-Food / 841-Butter / 966-Sandwich / 1472-Fruit preserves / 2197-Peanut butter / \n",
"1449-Rubber band / \n",
"5-Dance / 115-Ballet / \n",
"359-Harry Potter (Literary Series) / \n",
"3-Concert / \n",
"8-Football / \n",
"2-Video game / 37-Gadget / 55-Video game console / 292-Handheld game console / 317-PlayStation / 335-Nintendo 3DS / 362-PlayStation Portable / 682-PlayStation Vita / \n",
"45-Cosmetics / 56-Hair / \n",
"641-Mask / \n",
"1-Vehicle / 4-Car / 4638-Nissan Micra / \n",
"31-Disc jockey / \n",
"3-Concert / 5-Dance / 7-Musician / 13-Musical ensemble / 16-Performance art / 46-Choir / 49-School / \n",
"56-Hair / 106-Hairstyle / 1271-Beauty salon / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"3-Concert / 5-Dance / 7-Musician / 150-Violin / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 114-Engine / 618-Fiat Automobiles / \n",
"1033-Drill / 3224-European perch / \n",
"10-Animal / 48-Pet / 351-Jumping / 404-Livestock / 987-Sheep / 1116-Goat / \n",
"0-Games / 59-Winter sport / 112-Ice skating / 142-Hockey / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 67-Cycling / 69-Bicycle / 105-Motorcycling / 154-Wheel / 608-Wheelie / \n",
"20-Trailer / \n",
"357-DVD / 3449-Box set / \n",
"12-Food / 26-Cooking / 32-Recipe / 226-Dessert / 232-Cake / 236-Baking / 475-Cheese / 1641-Cheesecake / \n",
"1-Vehicle / 18-Outdoor recreation / 22-Nature / 197-Trail / 213-Forest / 267-Mountain / 614-Hiking / 1203-Backpacking / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / 2825-Rum / \n",
"0-Games / 19-PC game / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / 100-Electric guitar / 1259-Fender Telecaster / \n",
"0-Games / 8-Football / \n",
"117-Boxing / \n",
"0-Games / 2-Video game / 20-Trailer / 3440-Serious Sam / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 85-Combat / 382-Dota 2 / \n",
"0-Games / 2-Video game / 116-Pokémon / 186-Pokémon / 2139-Dungeon crawl / 2465-Pokémon Mystery Dungeon / \n",
"6-Animation / 15-Cartoon / 877-Hatsune Miku: Project DIVA / \n",
"0-Games / 256-Tennis / 544-Table tennis / \n",
"0-Games / 35-Minecraft / 2035-Forge / \n",
"5-Dance / 46-Choir / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 207-Exhaust system / 212-Highway / 245-BMW / 344-Coupé / 874-BMW M3 / 1709-BMW 3 Series (E46) / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 73-Ball / \n",
"0-Games / 35-Minecraft / \n",
"0-Games / 49-School / 54-Highlight film / 60-Basketball / 168-Basketball moves / 306-High school / \n",
"122-Weight training / 204-Gym / 214-Muscle / 529-Squat / 532-Barbell / 1584-Kettlebell / 2044-Bodyweight exercise / \n",
"6-Animation / 25-Toy / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / 265-Electronic keyboard / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 1056-Porsche 911 / \n",
"0-Games / 60-Basketball / 77-Arena / 81-Athlete / 168-Basketball moves / 255-Slam dunk / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"25-Toy / 1302-The Lego Group / \n",
"1-Vehicle / 4-Car / 11-Motorsport / \n",
"1-Vehicle / 18-Outdoor recreation / 22-Nature / 67-Cycling / 69-Bicycle / 197-Trail / 213-Forest / 220-Mountain bike / 284-Mountain biking / \n",
"0-Games / 51-Strategy video game / 165-League of Legends / 356-Star / \n",
"6-Animation / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"22-Nature / 86-Plant / 136-Gardening / 184-Vegetable / 279-Garden / 642-Leaf / \n",
"3-Concert / \n",
"1-Vehicle / 22-Nature / 147-River / 223-Weather / 283-Rock / 490-Rain / \n",
"7-Musician / 9-Music video / \n",
"110-Album / \n",
"9-Music video / 402-Jewellery / 587-Clay / 907-Necklace / 1300-Earrings / 1369-Necktie / \n",
"22-Nature / \n",
"8-Football / \n",
"1-Vehicle / 2-Video game / 4-Car / 70-Driving / 194-Call of Duty: Modern Warfare 2 / \n",
"9-Music video / \n",
"1-Vehicle / 50-Aircraft / 266-Helicopter / 1409-Missile / \n",
"1-Vehicle / 18-Outdoor recreation / 22-Nature / 67-Cycling / 69-Bicycle / 220-Mountain bike / 284-Mountain biking / \n",
"0-Games / 2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / 821-Rail freight transport / \n",
"6-Animation / 15-Cartoon / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"5-Dance / 16-Performance art / \n",
"76-Ball / 2389-Exercise ball / \n",
"44-Drums / 3465-Davul / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 230-Brass instrument / 427-Trumpet / \n",
"2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 101-Telephone / 508-iPhone 5 / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 531-Hatchback / 683-Renault / \n",
"1-Vehicle / 25-Toy / 74-Truck / 88-Machine / 822-Crane / 2632-Crusher / \n",
"22-Nature / 86-Plant / 136-Gardening / 184-Vegetable / 279-Garden / 363-Soil / 1863-Fertilizer / 2138-Compost / 2218-Manure / \n",
"9-Music video / \n",
"3-Concert / \n",
"878-Composer / \n",
"60-Basketball / 81-Athlete / \n",
"10-Animal / 48-Pet / 71-Dog / 84-Snow / 120-Winter / 1215-Wild boar / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 94-Dashcam / 790-Nissan GT-R / \n",
"66-Bollywood / \n",
"9-Music video / \n",
"3-Concert / \n",
"31-Disc jockey / \n",
"1-Vehicle / 11-Motorsport / 88-Machine / 114-Engine / 124-Tractor / 252-Tractor pulling / \n",
"1-Vehicle / 42-Fishing / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / 2946-Sonic the Hedgehog / \n",
"9-Music video / \n",
"28-Fashion / 45-Cosmetics / \n",
"36-Piano / 53-Keyboard / 646-Pipe organ / \n",
"12-Food / 26-Cooking / 32-Recipe / 109-Cooking show / 236-Baking / 442-Bread / 485-Dough / 1693-Pudding / 2125-Fillet / 3003-Roll / \n",
"49-School / 410-Teacher / 745-Writing / 845-Classroom / \n",
"66-Bollywood / \n",
"0-Games / 240-Arcade game / 514-Tekken / \n",
"130-Gymnastics / 2322-Horizontal bar / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 210-Cookware and bakeware / 236-Baking / 239-Roasting / \n",
"22-Nature / 1531-Sunrise / 1625-Sunset / \n",
"1-Vehicle / 18-Outdoor recreation / 22-Nature / 137-Tree / 213-Forest / 825-Stream / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / \n",
"28-Fashion / 206-Shoe / 785-Jeans / \n",
"162-Medicine / 2311-Human tooth / \n",
"99-Christmas / \n",
"1-Vehicle / 4-Car / 299-Volkswagen Passenger Cars / \n",
"151-Landing / \n",
"20-Trailer / \n",
"3-Concert / 68-Lighting / \n",
"10-Animal / 48-Pet / 71-Dog / 166-Cat / 346-Kitten / \n",
"707-Roof / 2729-Balcony / \n",
"5-Dance / \n",
"5-Dance / 16-Performance art / 350-Circus / \n",
"0-Games / 411-MapleStory / \n",
"7-Musician / 219-Accordion / \n",
"36-Piano / 53-Keyboard / 61-Art / 97-Drawing / 148-Painting / 528-Pencil / 628-Portrait / \n",
"3-Concert / \n",
"1-Vehicle / 62-Train / 64-Transport / 102-Building / 144-Amusement park / 161-Train station / 224-Rapid transit / 359-Harry Potter (Literary Series) / \n",
"10-Animal / 25-Toy / 524-Littlest Pet Shop / \n",
"20-Trailer / 325-Vampire / \n",
"0-Games / 192-Battlefield / 1370-Battlefield 2 / \n",
"1-Vehicle / 87-Boat / 147-River / 481-Kayak / \n",
"12-Food / \n",
"1-Vehicle / 64-Transport / 74-Truck / 1046-Remote control / \n",
"28-Fashion / 45-Cosmetics / \n",
"5-Dance / 16-Performance art / 783-Zee TV / \n",
"23-Mobile phone / 29-Smartphone / 345-Loudspeaker / 465-Microphone / \n",
"0-Games / 2-Video game / 1482-PlayStation Network / 2006-BlazBlue / 3627-BlazBlue: Continuum Shift / \n",
"0-Games / 35-Minecraft / \n",
"0-Games / 8-Football / 90-Sports game / 180-FIFA 15 / 623-Coach / \n",
"10-Animal / 48-Pet / 132-Bird / \n",
"205-Newscaster / 780-Fat / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 1338-Devil / \n",
"512-Printing / 525-Printer / 1063-Ink / \n",
"3-Concert / 133-Nightclub / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 30-Drummer / \n",
"56-Hair / 106-Hairstyle / 405-Braid / 1064-French braid / 1292-Ponytail / 2456-Tail / \n",
"0-Games / 2-Video game / 51-Strategy video game / 55-Video game console / 199-Wii / \n",
"1-Vehicle / 4-Car / 3138-Pressure washer / \n",
"15-Cartoon / 185-Dragon Ball / 244-Goku / \n",
"5-Dance / 130-Gymnastics / 3830-Double Dutch / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / 265-Electronic keyboard / \n",
"25-Toy / 140-LEGO / \n",
"5-Dance / \n",
"75-Wrestling / \n",
"5-Dance / 16-Performance art / 95-Talent show / \n",
"3-Concert / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 247-Stallion / 289-Dressage / 404-Livestock / 1434-Saddle / \n",
"0-Games / 2-Video game / 51-Strategy video game / 373-Clash of Clans / \n",
"223-Weather / \n",
"56-Hair / 106-Hairstyle / \n",
"5-Dance / 44-Drums / 98-Festival / \n",
"31-Disc jockey / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / \n",
"113-House / 329-Swimming pool / 369-Resort / \n",
"1640-Library / \n",
"402-Jewellery / 1010-Ring / 1300-Earrings / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 1355-Vegetarian food / \n",
"162-Medicine / \n",
"1-Vehicle / 4-Car / 179-Kitchen / 519-Recreational vehicle / 742-Countertop / \n",
"7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / 44-Drums / \n",
"15-Cartoon / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"59-Winter sport / 112-Ice skating / 316-Figure skating / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / \n",
"0-Games / 59-Winter sport / 81-Athlete / 112-Ice skating / 142-Hockey / 312-Ice rink / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"18-Outdoor recreation / 144-Amusement park / 321-Roller coaster / 334-Amusement ride / \n",
"0-Games / 2-Video game / 320-Super Smash Bros. / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 2-Video game / 60-Basketball / 90-Sports game / \n",
"0-Games / 1638-Soulcalibur / 2633-Soulcalibur V / \n",
"0-Games / 85-Combat / 4690-Shadow Fighter / \n",
"0-Games / 2-Video game / 505-Mega Man / 1664-Zero / 2474-Mega Man Battle Network / \n",
"31-Disc jockey / 227-Water / \n",
"1-Vehicle / 4-Car / \n",
"1-Vehicle / 11-Motorsport / 74-Truck / 124-Tractor / 135-Drifting / 249-Heavy equipment / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 70-Driving / 901-Mitsubishi / 3047-Colin McRae: Dirt 2 / 3136-Mitsubishi Eclipse / 3500-Colin McRae: Dirt / \n",
"0-Games / 2-Video game / 55-Video game console / 586-Nintendo Entertainment System / 910-Dragon Quest / \n",
"56-Hair / 106-Hairstyle / 1886-Bob cut / \n",
"9-Music video / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / \n",
"372-Playing card / 746-Card manipulation / \n",
"1-Vehicle / 4-Car / 207-Exhaust system / 299-Volkswagen Passenger Cars / 526-Volkswagen / 668-Volkswagen Golf / \n",
"59-Winter sport / 84-Snow / 177-Skiing / 789-Alpine skiing / \n",
"0-Games / 2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 201-Grand Theft Auto: San Andreas / 3307-HTC One X / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / \n",
"12-Food / 1823-Espresso machine / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"0-Games / 6-Animation / 269-Comic book / 460-Yu-Gi-Oh! Trading Card Game / \n",
"9-Music video / 14-Guitar / \n",
"46-Choir / \n",
"0-Games / 59-Winter sport / 81-Athlete / 112-Ice skating / 142-Hockey / 262-Ice / 312-Ice rink / 1973-Hockey puck / \n",
"6-Animation / 158-Tire / \n",
"46-Choir / 446-Sheet music / \n",
"31-Disc jockey / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 42-Fishing / 147-River / 225-Lake / 268-Recreational fishing / 825-Stream / 895-Trout / 1873-Crossbow / 3145-Spinnerbait / \n",
"156-Agriculture / 1579-Circle / 3483-Crop circle / \n",
"98-Festival / 209-University / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 169-Rallying / \n",
"60-Basketball / 76-Ball / 168-Basketball moves / 1101-Point guard / 3291-Basketball Coach / \n",
"6-Animation / 15-Cartoon / \n",
"10-Animal / 17-Racing / 80-Horse / 384-Horse racing / \n",
"1-Vehicle / 12-Food / 88-Machine / \n",
"35-Minecraft / \n",
"49-School / \n",
"1944-Vatican City / \n",
"3-Concert / 5-Dance / 16-Performance art / 153-Dress / \n",
"14-Guitar / 63-Acoustic guitar / 100-Electric guitar / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 121-Photography / 155-Camera / 629-Camera lens / 4431-Camera phone / \n",
"43-Call of Duty / \n",
"9-Music video / 920-Chair / \n",
"10-Animal / 91-Fish / 2090-Siamese fighting fish / \n",
"59-Winter sport / 112-Ice skating / 142-Hockey / \n",
"9-Music video / \n",
"657-Autumn / 1459-Magazine / 1545-Floristry / \n",
"0-Games / 256-Tennis / 462-Table / 544-Table tennis / 1337-Serve / 1615-Pong / \n",
"147-River / \n",
"3-Concert / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 554-iPhone 4 / \n",
"10-Animal / 48-Pet / 71-Dog / 706-Deer / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / \n",
"3-Concert / 16-Performance art / \n",
"5-Dance / 46-Choir / \n",
"5-Dance / 125-Television / \n",
"310-The Sims / 543-The Sims 2 / \n",
"3-Concert / \n",
"61-Art / 148-Painting / 538-Illustration / \n",
"0-Games / 21-Stadium / 60-Basketball / 77-Arena / 168-Basketball moves / 255-Slam dunk / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 205-Newscaster / \n",
"0-Games / 10-Animal / 48-Pet / \n",
"3-Concert / \n",
"3-Concert / 133-Nightclub / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 17-Racing / 34-Action-adventure game / 111-PlayStation 3 / 118-Grand Theft Auto V / \n",
"766-President of the United States / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 40-Road / 94-Dashcam / 212-Highway / 245-BMW / 1622-BMW M5 / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 57-Race track / 70-Driving / 94-Dashcam / 212-Highway / 2214-Multi-valve / 4352-Citroën Saxo / \n",
"3-Concert / 2765-Mail / \n",
"1-Vehicle / 4-Car / 70-Driving / 618-Fiat Automobiles / 864-Speedometer / \n",
"3-Concert / 7-Musician / 16-Performance art / 30-Drummer / 39-Drums / 44-Drums / \n",
"0-Games / 460-Yu-Gi-Oh! Trading Card Game / \n",
"93-Comedy / 355-Comedian / \n",
"6-Animation / 67-Cycling / 69-Bicycle / 126-The Walt Disney Company / \n",
"28-Fashion / 880-Trousers / 986-Suit / 2188-Coat / 3781-Hoodie / 4505-GQ / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"10-Animal / 48-Pet / 166-Cat / 346-Kitten / \n",
"0-Games / 59-Winter sport / 81-Athlete / 112-Ice skating / 142-Hockey / \n",
"6-Animation / \n",
"6-Animation / 3689-Woody Woodpecker / \n",
"5-Dance / 95-Talent show / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 101-Telephone / 178-Samsung Galaxy / 512-Printing / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 2-Video game / 33-Weapon / 176-PlayStation 4 / 948-Far Cry / 1683-Far Cry 4 / \n",
"18-Outdoor recreation / 315-Knife / 1970-Axe / \n",
"159-World of Warcraft / 171-Warcraft / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"619-Manufacturing / 1363-Candle / \n",
"5-Dance / 115-Ballet / \n",
"1-Vehicle / 4-Car / 167-Four-wheel drive / 198-Off-road vehicle / 213-Forest / \n",
"497-Macintosh / 512-Printing / 2711-Label / \n",
"20-Trailer / \n",
"67-Cycling / 69-Bicycle / 204-Gym / 284-Mountain biking / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / \n",
"1-Vehicle / 4-Car / 245-BMW / 983-Lexus / 1622-BMW M5 / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 40-Road / 67-Cycling / 69-Bicycle / 105-Motorcycling / \n",
"0-Games / 8-Football / \n",
"425-City / \n",
"12-Food / 26-Cooking / 184-Vegetable / \n",
"1-Vehicle / 4-Car / 10-Animal / 132-Bird / 598-Parrot / \n",
"313-Book / \n",
"0-Games / 2-Video game / 1170-Super Mario World / 1383-Yoshi / 2589-Yoshi's Island / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 39-Drums / \n",
"0-Games / 159-World of Warcraft / 171-Warcraft / \n",
"1382-Half-Life 2 / 1412-Half-Life / \n",
"28-Fashion / 206-Shoe / 757-Boot / \n",
"191-Cheerleading / 293-Slide show / \n",
"9-Music video / 78-Wedding / 190-Bride / \n",
"59-Winter sport / 84-Snow / 120-Winter / 177-Skiing / 305-Ski / \n",
"3-Concert / \n",
"5-Dance / 115-Ballet / 430-Ballroom dance / 678-Latin dance / \n",
"51-Strategy video game / 774-Galaxy / 1718-Spore / \n",
"98-Festival / 253-Carnival / \n",
"0-Games / 1-Vehicle / 4-Car / 17-Racing / \n",
"10-Animal / \n",
"28-Fashion / \n",
"1-Vehicle / 11-Motorsport / 18-Outdoor recreation / 27-Motorcycle / 42-Fishing / 1025-Personal water craft / 1336-Jet Ski / \n",
"6-Animation / 15-Cartoon / \n",
"20-Trailer / \n",
"0-Games / 2-Video game / 43-Call of Duty / 194-Call of Duty: Modern Warfare 2 / \n",
"1-Vehicle / 11-Motorsport / \n",
"47-Personal computer / 131-Computer / 309-Computer hardware / 602-Central processing unit / \n",
"0-Games / 2-Video game / 20-Trailer / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 150-Violin / \n",
"61-Art / 148-Painting / \n",
"6-Animation / 15-Cartoon / 183-Manga / 708-Mermaid / \n",
"121-Photography / 155-Camera / 629-Camera lens / \n",
"6-Animation / 25-Toy / 319-Robot / 477-Transformers / 990-Optimus Prime / \n",
"88-Machine / 512-Printing / 525-Printer / 536-Glass / 2634-Inkjet printing / \n",
"164-Beach / 191-Cheerleading / \n",
"18-Outdoor recreation / 144-Amusement park / 321-Roller coaster / 334-Amusement ride / \n",
"8-Football / \n",
"0-Games / 35-Minecraft / 1027-Terraria / \n",
"3391-Alesis / \n",
"1-Vehicle / 4-Car / 40-Road / 64-Transport / 70-Driving / \n",
"12-Food / \n",
"170-Home improvement / 246-Furniture / 276-Room / 2385-Closet / \n",
"0-Games / 2-Video game / 20-Trailer / 34-Action-adventure game / 1144-Castlevania / 3768-Castlevania: Lords of Shadow / \n",
"8-Football / \n",
"5-Dance / 49-School / 95-Talent show / 191-Cheerleading / \n",
"0-Games / 1754-Rocksmith / 2050-Rocksmith 2014 / \n",
"0-Games / 19-PC game / 35-Minecraft / 2956-Furnace / \n",
"61-Art / 479-Graffiti / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / \n",
"24-String instrument / 2980-Sitar / \n",
"8-Football / 49-School / 79-American football / 306-High school / \n",
"5-Dance / \n",
"6-Animation / 51-Strategy video game / 269-Comic book / 1915-Duel Masters Trading Card Game / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 422-Call of Duty: Ghosts / 431-Call of Duty: Advanced Warfare / \n",
"1-Vehicle / 88-Machine / \n",
"31-Disc jockey / \n",
"214-Muscle / \n",
"5-Dance / 3240-Matryoshka doll / \n",
"2-Video game / 6-Animation / \n",
"6-Animation / \n",
"1-Vehicle / 11-Motorsport / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 47-Personal computer / 118-Grand Theft Auto V / \n",
"0-Games / 2-Video game / 33-Weapon / 420-The Elder Scrolls V: Skyrim / 441-The Elder Scrolls / 1130-Armour / \n",
"341-Sketch comedy / \n",
"15-Cartoon / 123-Naruto / 285-Sasuke Uchiha / \n",
"141-Microsoft Windows / 504-Web page / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 90-Sports game / \n",
"0-Games / 199-Wii / 456-Go-kart / 518-Mario Kart / \n",
"99-Christmas / \n",
"467-Chipmunk / 846-Squirrel / \n",
"1-Vehicle / 4-Car / 270-Sedan / 517-Audi / 1587-Audi Quattro / 2306-Audi A6 / \n",
"31-Disc jockey / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / \n",
"12-Food / 32-Recipe / 109-Cooking show / 226-Dessert / 1223-Pastry / \n",
"0-Games / 2-Video game / 19-PC game / 211-RuneScape / \n",
"130-Gymnastics / 1030-Trampoline / 1335-Trampolining / 2161-Playground / \n",
"1-Vehicle / 4-Car / 25-Toy / 88-Machine / 319-Robot / \n",
"18-Outdoor recreation / 51-Strategy video game / 144-Amusement park / 321-Roller coaster / 334-Amusement ride / 1293-RollerCoaster Tycoon 3 / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / 219-Accordion / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 57-Race track / 169-Rallying / \n",
"2393-Didgeridoo / \n",
"8-Football / 54-Highlight film / \n",
"380-Assassin's Creed / \n",
"6-Animation / 849-Tales / 3519-Tales of the Abyss / \n",
"162-Medicine / \n",
"6-Animation / 49-School / 102-Building / 113-House / 1246-Campus / \n",
"2-Video game / 35-Minecraft / 380-Assassin's Creed / 1048-Assassin's Creed / 1140-Assassin's Creed III / \n",
"0-Games / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 903-Mass Effect / 1655-Mass Effect 2 / \n",
"0-Games / 47-Personal computer / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / \n",
"18-Outdoor recreation / 227-Water / 287-Human swimming / 329-Swimming pool / \n",
"67-Cycling / 69-Bicycle / 570-Road bicycle racing / 645-Road bicycle / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 237-Final Fantasy / 1571-Final Fantasy XIV / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 165-League of Legends / 1494-Jungle / \n",
"47-Personal computer / 53-Keyboard / 145-Tablet computer / 513-Computer keyboard / 1565-Logitech / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / 472-Boeing 737 / 719-Boeing 747 / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 989-Warface / \n",
"3-Concert / 7-Musician / \n",
"1-Vehicle / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 232-Cake / 500-Rice / 2591-Filipino food / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 19-PC game / 34-Action-adventure game / 111-PlayStation 3 / 413-PlayStation 2 / 921-Grand Theft Auto: Vice City / 2893-Grand Theft Auto: Vice City Stories / \n",
"8-Football / 21-Stadium / \n",
"1-Vehicle / 4-Car / 1742-Lowrider / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 220-Mountain bike / 284-Mountain biking / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 570-Road bicycle racing / 645-Road bicycle / \n",
"1208-Coconut / \n",
"10-Animal / 12-Food / 71-Dog / 232-Cake / 323-Toddler / \n",
"3-Concert / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"474-Destiny / \n",
"6-Animation / 15-Cartoon / 185-Dragon Ball / 244-Goku / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"8-Football / \n",
"9-Music video / 31-Disc jockey / \n",
"1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 55-Video game console / 57-Race track / 111-PlayStation 3 / \n",
"83-Skateboarding / 108-Skateboard / \n",
"6-Animation / 137-Tree / 1738-Elmo / \n",
"0-Games / 2-Video game / 55-Video game console / 413-PlayStation 2 / 2486-Scooby-Doo / \n",
"3-Concert / 5-Dance / 13-Musical ensemble / 16-Performance art / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"6-Animation / 15-Cartoon / 1594-Nerd / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 96-Soldier / 638-Gears of War / 773-Gears of War / 1073-Chainsaw / 1425-Gears of War 2 / \n",
"313-Book / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"22-Nature / 1151-Temple / 3039-Wat / \n",
"8-Football / \n",
"0-Games / 1394-Alliance of Valiant Arms / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 306-High school / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 378-Drag racing / 482-Asphalt / \n",
"3544-Rubber stamp / \n",
"3-Concert / 68-Lighting / 1712-Sweater / \n",
"5-Dance / 44-Drums / 115-Ballet / \n",
"1-Vehicle / 4-Car / 64-Transport / 286-Bus / \n",
"45-Cosmetics / 49-School / 56-Hair / \n",
"28-Fashion / 206-Shoe / 368-Sneakers / 2014-Vans / 3831-Supreme / \n",
"695-Balloon / \n",
"1-Vehicle / 4-Car / 114-Engine / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 73-Ball / \n",
"6-Animation / 15-Cartoon / 1492-Persona / \n",
"10-Animal / 15-Cartoon / 61-Art / 71-Dog / 97-Drawing / 408-Sketch / 528-Pencil / \n",
"0-Games / 2-Video game / 35-Minecraft / 394-Map / \n",
"31-Disc jockey / \n",
"10-Animal / 48-Pet / 71-Dog / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"1-Vehicle / 4-Car / 449-Honda / 519-Recreational vehicle / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / \n",
"3-Concert / 16-Performance art / \n",
"5-Dance / 16-Performance art / 95-Talent show / \n",
"6-Animation / 15-Cartoon / 116-Pokémon / 1104-Ash Ketchum / \n",
"125-Television / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"5-Dance / 115-Ballet / \n",
"640-Bottle / 1175-Soap / 4325-Detergent / \n",
"0-Games / 2-Video game / 34-Action-adventure game / \n",
"445-Guitar Hero / 965-Guitar Hero III: Legends of Rock / \n",
"10-Animal / 20-Trailer / 550-Dinosaur / 1126-Tyrannosaurus / \n",
"0-Games / 8-Football / 54-Highlight film / 73-Ball / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"5-Dance / 16-Performance art / \n",
"206-Shoe / \n",
"18-Outdoor recreation / 42-Fishing / 67-Cycling / 69-Bicycle / \n",
"778-Gliding / 1602-Hang gliding / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 179-Kitchen / 295-Chocolate / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 1820-Motorola Droid / \n",
"31-Disc jockey / \n",
"28-Fashion / \n",
"13-Musical ensemble / \n",
"1-Vehicle / 4-Car / \n",
"8-Football / 21-Stadium / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"8-Football / \n",
"22-Nature / \n",
"122-Weight training / 214-Muscle / \n",
"0-Games / 8-Football / 21-Stadium / 79-American football / \n",
"3-Concert / \n",
"28-Fashion / 45-Cosmetics / 56-Hair / 106-Hairstyle / 340-Afro-textured hair / 857-Afro / 909-Updo / 1320-Bangs / \n",
"24-String instrument / 502-Sky / 842-Harp / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / 3637-Steel-string acoustic guitar / \n",
"14-Guitar / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 482-Asphalt / \n",
"12-Food / 812-Plastic / 1870-Chewing gum / \n",
"6-Animation / 15-Cartoon / 290-Pony / 343-My Little Pony / \n",
"1-Vehicle / 74-Truck / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 105-Motorcycling / 172-Motocross / \n",
"3-Concert / 28-Fashion / 495-Runway / \n",
"75-Wrestling / 85-Combat / \n",
"13-Musical ensemble / 356-Star / \n",
"5-Dance / \n",
"6-Animation / 25-Toy / 749-Figurine / \n",
"215-Eating / 534-Milk / 2080-Cup / \n",
"3-Concert / \n",
"0-Games / 8-Football / \n",
"0-Games / 6-Animation / 995-Attack on Titan / \n",
"8-Football / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / \n",
"0-Games / 2-Video game / 229-Halo / 1161-Halo 2 / \n",
"5-Dance / 46-Choir / 318-Church / \n",
"6-Animation / 25-Toy / 323-Toddler / 3785-Crocodilia / \n",
"2077-Chemical reaction / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / \n",
"28-Fashion / 469-Bag / 676-Handbag / 2188-Coat / \n",
"337-Advertising / \n",
"9-Music video / \n",
"223-Weather / \n",
"1-Vehicle / 11-Motorsport / 88-Machine / 124-Tractor / 156-Agriculture / 174-Farm / 252-Tractor pulling / \n",
"7-Musician / 13-Musical ensemble / 230-Brass instrument / 925-Brain / \n",
"1-Vehicle / 4-Car / 41-Sports car / 114-Engine / 245-BMW / \n",
"59-Winter sport / 112-Ice skating / 262-Ice / 316-Figure skating / 833-Ice skate / \n",
"0-Games / 2-Video game / 33-Weapon / 635-Achievement / 924-Trophy / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 184-Vegetable / 572-Indian cuisine / 597-Soup / 727-Curry / 1509-Stew / 2000-Mango / \n",
"1-Vehicle / 4-Car / 41-Sports car / \n",
"20-Trailer / 886-James Bond / \n",
"54-Highlight film / 60-Basketball / \n",
"0-Games / 2-Video game / 4499-Fable II / \n",
"25-Toy / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / 44-Drums / \n",
"31-Disc jockey / \n",
"0-Games / 2-Video game / 8-Football / 90-Sports game / 180-FIFA 15 / \n",
"3-Concert / 5-Dance / 16-Performance art / 490-Rain / \n",
"1-Vehicle / 27-Motorcycle / \n",
"31-Disc jockey / \n",
"6-Animation / 1454-Stairs / \n",
"0-Games / 8-Football / \n",
"2-Video game / 6-Animation / \n",
"3-Concert / 7-Musician / \n",
"164-Beach / 233-Ocean / \n",
"110-Album / 567-Compact disc / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / \n",
"219-Accordion / \n",
"2-Video game / 2277-The Lord of the Rings Online / \n",
"24-String instrument / 150-Violin / 401-Cello / \n",
"0-Games / 2-Video game / 241-Fighting game / 2301-Dead or Alive / 4630-Dead or Alive 5 Ultimate / \n",
"1-Vehicle / 4-Car / \n",
"10-Animal / 48-Pet / 132-Bird / \n",
"31-Disc jockey / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 224-Rapid transit / 794-Bridge / \n",
"6-Animation / 20-Trailer / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 101-Telephone / 675-iPhone 4S / \n",
"75-Wrestling / \n",
"6-Animation / 15-Cartoon / 116-Pokémon / 186-Pokémon / 1104-Ash Ketchum / \n",
"75-Wrestling / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 212-Highway / 299-Volkswagen Passenger Cars / 482-Asphalt / 668-Volkswagen Golf / 2286-Grand Tourer Injection / 3839-Volkswagen Golf Mk5 / \n",
"6-Animation / 538-Illustration / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 102-Building / 129-Aviation / 139-Airport / 151-Landing / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / 856-Airbus A320 family / 923-Boeing 777 / \n",
"53-Keyboard / 230-Brass instrument / 3391-Alesis / \n",
"257-Counter-Strike / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 245-BMW / 491-Enduro / 4403-BMW F series parallel-twin / \n",
"1-Vehicle / 4-Car / 41-Sports car / \n",
"3-Concert / 13-Musical ensemble / 30-Drummer / 44-Drums / 253-Carnival / \n",
"0-Games / 35-Minecraft / \n",
"101-Telephone / 3233-Screencast / \n",
"45-Cosmetics / \n",
"3-Concert / \n",
"13-Musical ensemble / 44-Drums / \n",
"5-Dance / \n",
"31-Disc jockey / 861-Mixing console / 2193-CDJ / \n",
"8-Football / \n",
"7-Musician / 13-Musical ensemble / 98-Festival / 203-Marching band / 250-Parade / \n",
"3-Concert / \n",
"1-Vehicle / 88-Machine / 114-Engine / \n",
"1-Vehicle / 164-Beach / 423-Coast / 1013-Walking / \n",
"141-Microsoft Windows / \n",
"1-Vehicle / 27-Motorcycle / 663-Street / \n",
"5-Dance / 16-Performance art / 98-Festival / 678-Latin dance / \n",
"0-Games / 54-Highlight film / 60-Basketball / \n",
"45-Cosmetics / 49-School / 193-Eye liner / 195-Mascara / 196-Eye / \n",
"7-Musician / 14-Guitar / 38-Orchestra / \n",
"59-Winter sport / 281-Snowboarding / \n",
"0-Games / 2054-Diablo II / \n",
"2-Video game / \n",
"8-Football / \n",
"1-Vehicle / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"59-Winter sport / 177-Skiing / 1764-Ski jumping / \n",
"10-Animal / 48-Pet / 166-Cat / 346-Kitten / 2211-Fur / 2924-Persian cat / \n",
"14-Guitar / \n",
"387-Spider-Man / \n",
"0-Games / 19-PC game / 35-Minecraft / 272-iPod / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / \n",
"355-Comedian / \n",
"0-Games / 2-Video game / 116-Pokémon / 186-Pokémon / 1387-Pokémon Battle Revolution / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 184-Vegetable / 239-Roasting / 500-Rice / 572-Indian cuisine / 727-Curry / 1355-Vegetarian food / 1509-Stew / 2363-Fried rice / 4236-Biryani / \n",
"5-Dance / 16-Performance art / \n",
"9-Music video / \n",
"115-Ballet / 1253-Ballet Dancer / \n",
"9-Music video / \n",
"0-Games / 366-Dragon / \n",
"0-Games / 2-Video game / 216-Call of Duty: Modern Warfare 3 / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 81-Athlete / \n",
"0-Games / 159-World of Warcraft / 171-Warcraft / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"60-Basketball / 255-Slam dunk / \n",
"6-Animation / 15-Cartoon / \n",
"21-Stadium / 85-Combat / 117-Boxing / \n",
"78-Wedding / 190-Bride / 421-Gown / \n",
"6-Animation / 238-Nail / 300-Nail art / 343-My Little Pony / 823-Artificial nails / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / \n",
"3-Concert / \n",
"3-Concert / 7-Musician / \n",
"3-Concert / 7-Musician / \n",
"49-School / \n",
"10-Animal / 468-Biology / \n",
"15-Cartoon / 200-Doll / 412-Monster / 503-Monster High / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 114-Engine / 234-Ford / 473-Ford Mustang / \n",
"7-Musician / 14-Guitar / \n",
"14-Guitar / 219-Accordion / 296-Saxophone / \n",
"31-Disc jockey / \n",
"34-Action-adventure game / 506-Garry's Mod / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 179-Kitchen / 184-Vegetable / 210-Cookware and bakeware / 239-Roasting / 274-Meat / 2125-Fillet / \n",
"12-Food / 1060-Raw food / \n",
"0-Games / 1167-Tanki Online / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 70-Driving / 207-Exhaust system / 2787-Saab Automobile AB / 4008-Saab / \n",
"1-Vehicle / 22-Nature / 50-Aircraft / 139-Airport / 151-Landing / 425-City / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / \n",
"93-Comedy / \n",
"42-Fishing / \n",
"1-Vehicle / 50-Aircraft / \n",
"1-Vehicle / 11-Motorsport / 124-Tractor / 156-Agriculture / 174-Farm / 252-Tractor pulling / 459-Plough / 906-Sowing / \n",
"0-Games / 2-Video game / 1288-Marvel vs. Capcom / 1499-Marvel vs. Capcom 3: Fate of Two Worlds / 1505-Ultimate Marvel vs. Capcom 3 / \n",
"3-Concert / \n",
"1-Vehicle / 11-Motorsport / 87-Boat / 351-Jumping / 1025-Personal water craft / 1076-Wakeboarding / 2282-Waterskiing / \n",
"0-Games / 2-Video game / 17-Racing / 1137-Crash Bandicoot / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 522-Helmet / 1903-Motorcycle helmet / \n",
"93-Comedy / 355-Comedian / \n",
"3-Concert / 13-Musical ensemble / 68-Lighting / \n",
"31-Disc jockey / \n",
"10-Animal / 166-Cat / 280-Wildlife / 440-Lion / 537-Zoo / 1039-Tiger / 1591-Leopard / \n",
"3-Concert / 5-Dance / \n",
"22-Nature / 559-Night / 654-Sun / 998-Landscape / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 105-Motorcycling / 172-Motocross / 197-Trail / 198-Off-road vehicle / 491-Enduro / \n",
"8-Football / 18-Outdoor recreation / 42-Fishing / \n",
"5-Dance / 98-Festival / 1289-Caporales / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 111-PlayStation 3 / 118-Grand Theft Auto V / \n",
"3-Concert / 5-Dance / 38-Orchestra / \n",
"3-Concert / 5-Dance / 16-Performance art / 95-Talent show / \n",
"6-Animation / \n",
"0-Games / 2-Video game / 849-Tales / \n",
"457-Knitting / 1804-Sewing needle / \n",
"0-Games / 159-World of Warcraft / 171-Warcraft / 1356-Paladin / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 70-Driving / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 213-Forest / 217-Hunting / 706-Deer / 1750-White-tailed deer / 2135-Bowhunting / \n",
"609-Tattoo / \n",
"14-Guitar / 738-Guitar amplifier / 4104-Fender Precision Bass / \n",
"28-Fashion / 206-Shoe / 785-Jeans / 1811-Forever 21 / 2315-Top / \n",
"1-Vehicle / 4-Car / 40-Road / 41-Sports car / 1763-Lotus Cars / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 422-Call of Duty: Ghosts / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 60-Basketball / 991-NBA 2K15 / 2156-Rookie / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / 1599-Bone / \n",
"3-Concert / 7-Musician / 36-Piano / 104-Pianist / \n",
"2-Video game / 269-Comic book / 950-Blu-ray disc / \n",
"0-Games / 2-Video game / 22-Nature / 86-Plant / 1410-The Elder Scrolls IV: Oblivion / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"1-Vehicle / 11-Motorsport / 351-Jumping / 794-Bridge / \n",
"1-Vehicle / 74-Truck / 1070-Semi-trailer truck / \n",
"31-Disc jockey / \n",
"0-Games / 2387-Military tactics / \n",
"22-Nature / 86-Plant / \n",
"0-Games / 2-Video game / 311-Street Fighter / 461-Street Fighter IV / 496-Super Street Fighter IV / \n",
"13-Musical ensemble / \n",
"0-Games / 2-Video game / 1566-Doom / 3440-Serious Sam / 3527-Doom 3 / 4556-Serious Sam / \n",
"222-Surfing / 1008-Kite / \n",
"2-Video game / 20-Trailer / 159-World of Warcraft / 171-Warcraft / 2828-World of Warcraft: The Burning Crusade / \n",
"3-Concert / 38-Orchestra / \n",
"0-Games / 47-Personal computer / 2265-Subway Surfers / \n",
"5-Dance / 13-Musical ensemble / 16-Performance art / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"447-Sand / 751-Desert / \n",
"0-Games / 59-Winter sport / 112-Ice skating / 142-Hockey / 312-Ice rink / \n",
"1-Vehicle / 94-Dashcam / \n",
"25-Toy / 116-Pokémon / 269-Comic book / 1021-Booster pack / 1049-Collectible card game / 1062-Pokémon Trading Card Game / \n",
"1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 70-Driving / 169-Rallying / \n",
"35-Minecraft / 43-Call of Duty / 113-House / 194-Call of Duty: Modern Warfare 2 / 394-Map / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 94-Dashcam / 212-Highway / 271-Tank / \n",
"0-Games / 2-Video game / 55-Video game console / 199-Wii / \n",
"0-Games / 2-Video game / 47-Personal computer / 176-PlayStation 4 / 181-Xbox / 1251-The Witcher / 1401-The Witcher 3: Wild Hunt / 2752-God Mode / \n",
"5-Dance / \n",
"123-Naruto / 285-Sasuke Uchiha / 622-Sakura Haruno / \n",
"31-Disc jockey / \n",
"0-Games / 2-Video game / 6-Animation / 692-Link / 1857-Sephiroth / \n",
"819-World Wide Web / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 15-Cartoon / 320-Super Smash Bros. / 782-Super Smash Bros. Melee / \n",
"3-Concert / 16-Performance art / \n",
"208-Wood / 492-Floor / 1011-Tile / 1357-Ceramic / 2362-Porcelain / \n",
"1-Vehicle / 4-Car / 74-Truck / 86-Plant / 137-Tree / 167-Four-wheel drive / 349-Chevrolet / 585-Lawn / 854-Diesel engine / 1969-Shrub / \n",
"9-Music video / \n",
"0-Games / 8-Football / 21-Stadium / 77-Arena / \n",
"0-Games / 2-Video game / 47-Personal computer / 394-Map / 497-Macintosh / 506-Garry's Mod / \n",
"9-Music video / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 18-Outdoor recreation / 22-Nature / 27-Motorcycle / 105-Motorcycling / 172-Motocross / 2420-Audi TT / \n",
"5-Dance / 115-Ballet / \n",
"9-Music video / 22-Nature / 657-Autumn / \n",
"8-Football / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"3-Concert / 21-Stadium / 68-Lighting / \n",
"0-Games / 2-Video game / 23-Mobile phone / 29-Smartphone / \n",
"0-Games / 51-Strategy video game / 165-League of Legends / \n",
"5-Dance / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 247-Stallion / 289-Dressage / \n",
"0-Games / 8-Football / 54-Highlight film / 76-Ball / 81-Athlete / \n",
"9-Music video / 15-Cartoon / 412-Monster / 503-Monster High / \n",
"209-University / \n",
"36-Piano / 53-Keyboard / \n",
"9-Music video / 110-Album / \n",
"5-Dance / 98-Festival / 250-Parade / 253-Carnival / \n",
"0-Games / 14-Guitar / 211-RuneScape / 366-Dragon / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"365-Fireworks / \n",
"0-Games / 2-Video game / 33-Weapon / 779-Point Blank / \n",
"5-Dance / 6-Animation / 1523-Alice / 2201-Alice's Adventures in Wonderland / 3342-The Mad Hatter / \n",
"8-Football / 4270-Old Trafford / \n",
"47-Personal computer / 228-Laptop / 525-Printer / 2634-Inkjet printing / \n",
"12-Food / 26-Cooking / \n",
"0-Games / 1-Vehicle / 4-Car / 6-Animation / 15-Cartoon / 25-Toy / 74-Truck / 189-iPad / 433-Construction / 1429-Dump truck / \n",
"99-Christmas / 558-Santa Claus / \n",
"3-Concert / 30-Drummer / \n",
"0-Games / 211-RuneScape / \n",
"15-Cartoon / \n",
"219-Accordion / \n",
"1-Vehicle / 62-Train / 249-Heavy equipment / 433-Construction / 817-Bulldozer / 976-Demolition / \n",
"9-Music video / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 57-Race track / 67-Cycling / 69-Bicycle / 2003-Track cycling / \n",
"0-Games / 2-Video game / 6-Animation / 1343-Super Robot Wars / 3128-Super Robot / \n",
"9-Music video / \n",
"66-Bollywood / 1606-Butterfly / \n",
"46-Choir / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 235-Lipstick / 1012-Lip gloss / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / \n",
"7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / 3042-The Salvation Army / \n",
"390-Model / \n",
"0-Games / 2-Video game / 6-Animation / 241-Fighting game / 514-Tekken / 797-Tekken / 1470-Tekken Tag Tournament 2 / 4354-Ling Xiaoyu / \n",
"9-Music video / \n",
"1-Vehicle / 87-Boat / \n",
"5-Dance / 16-Performance art / 95-Talent show / \n",
"6-Animation / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"78-Wedding / \n",
"10-Animal / 91-Fish / 258-Aquarium / 288-Underwater / 601-Shark / \n",
"1-Vehicle / 4-Car / \n",
"6-Animation / 701-Beyblade / \n",
"162-Medicine / \n",
"439-Barbie / \n",
"319-Robot / 3256-Robotic vacuum cleaner / 4127-Roomba / \n",
"1-Vehicle / 153-Dress / \n",
"3-Concert / 133-Nightclub / 530-Cue sports / 613-Bar / 1280-Club / \n",
"18-Outdoor recreation / 59-Winter sport / 84-Snow / 120-Winter / 177-Skiing / 305-Ski / 501-Snowboard / 789-Alpine skiing / 1830-Downhill / \n",
"0-Games / 6-Animation / 15-Cartoon / 89-Comics / \n",
"3329-Upholstery / \n",
"3-Concert / 68-Lighting / \n",
"5-Dance / 115-Ballet / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / 1612-Multi Theft Auto / \n",
"22-Nature / \n",
"1411-ESPN / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 1240-Mambo / \n",
"0-Games / 18-Outdoor recreation / 54-Highlight film / 81-Athlete / 163-Running / 674-Marathon / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 41-Sports car / 70-Driving / 1391-Kia / 1415-Kia Motors / 4498-Kia Optima / \n",
"0-Games / 2-Video game / 1029-Aion: The Tower of Eternity / \n",
"1-Vehicle / 4-Car / 41-Sports car / 154-Wheel / 245-BMW / 874-BMW M3 / \n",
"689-Harmonica / \n",
"0-Games / 2-Video game / 411-MapleStory / \n",
"0-Games / \n",
"1-Vehicle / 27-Motorcycle / 207-Exhaust system / 970-Monkey / \n",
"0-Games / 2-Video game / 33-Weapon / 85-Combat / 96-Soldier / 989-Warface / \n",
"310-The Sims / 1553-The Sims 4 / \n",
"1-Vehicle / 22-Nature / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 267-Mountain / 800-Mountain pass / \n",
"0-Games / 8-Football / 54-Highlight film / 73-Ball / 90-Sports game / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"21-Stadium / 77-Arena / 79-American football / \n",
"114-Engine / 778-Gliding / 1602-Hang gliding / \n",
"1-Vehicle / 18-Outdoor recreation / 87-Boat / 2338-Inflatable boat / \n",
"5-Dance / 78-Wedding / 190-Bride / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 178-Samsung Galaxy / \n",
"330-Money / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 570-Road bicycle racing / 2003-Track cycling / \n",
"14-Guitar / 24-String instrument / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 166-Cat / 216-Call of Duty: Modern Warfare 3 / 302-Call of Duty 4: Modern Warfare / \n",
"5-Dance / \n",
"9-Music video / 31-Disc jockey / \n",
"125-Television / \n",
"9-Music video / 14-Guitar / \n",
"1-Vehicle / 4-Car / 901-Mitsubishi / 1413-Mitsubishi Lancer / \n",
"110-Album / \n",
"49-School / \n",
"5-Dance / 16-Performance art / 678-Latin dance / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / 265-Electronic keyboard / \n",
"22-Nature / 121-Photography / \n",
"20-Trailer / 359-Harry Potter (Literary Series) / \n",
"3-Concert / 5-Dance / \n",
"5-Dance / 16-Performance art / \n",
"12-Food / 26-Cooking / 52-Dish / 58-Cuisine / 109-Cooking show / 215-Eating / 225-Lake / 301-Meal / 771-Noodle / \n",
"28-Fashion / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / \n",
"0-Games / 2-Video game / 357-DVD / 413-PlayStation 2 / 581-PlayStation / \n",
"6-Animation / 10-Animal / 15-Cartoon / \n",
"949-Rowing / \n",
"6-Animation / 15-Cartoon / 61-Art / 97-Drawing / 196-Eye / 408-Sketch / 528-Pencil / \n",
"20-Trailer / 630-Superman / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"230-Brass instrument / 296-Saxophone / \n",
"9-Music video / \n",
"174-Farm / 523-Cattle / 660-Rural area / \n",
"5-Dance / 98-Festival / 191-Cheerleading / \n",
"6-Animation / 15-Cartoon / \n",
"10-Animal / 71-Dog / 83-Skateboarding / 108-Skateboard / \n",
"0-Games / 60-Basketball / 255-Slam dunk / \n",
"66-Bollywood / \n",
"365-Fireworks / \n",
"60-Basketball / 73-Ball / 76-Ball / \n",
"1-Vehicle / 27-Motorcycle / 67-Cycling / 69-Bicycle / 114-Engine / 1952-Carburetor / 3578-Motorized bicycle / \n",
"6-Animation / 15-Cartoon / \n",
"1-Vehicle / 4-Car / 270-Sedan / 403-Toyota / 1583-Toyota Corolla / \n",
"9-Music video / 14-Guitar / \n",
"98-Festival / \n",
"609-Tattoo / \n",
"3-Concert / 7-Musician / 14-Guitar / 30-Drummer / 39-Drums / \n",
"5-Dance / \n",
"3-Concert / 68-Lighting / \n",
"1-Vehicle / 4-Car / 2842-Drive-through / 2852-Burger King / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 67-Cycling / 69-Bicycle / 105-Motorcycling / \n",
"3-Concert / \n",
"6-Animation / 15-Cartoon / 185-Dragon Ball / 244-Goku / 418-Gohan / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 111-PlayStation 3 / 118-Grand Theft Auto V / \n",
"10-Animal / 12-Food / 91-Fish / \n",
"1-Vehicle / 4-Car / 74-Truck / 375-Pickup truck / 406-Chevrolet / \n",
"0-Games / 8-Football / 21-Stadium / 76-Ball / \n",
"350-Circus / 788-Juggling / \n",
"31-Disc jockey / \n",
"9-Music video / \n",
"5-Dance / 6-Animation / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 235-Lipstick / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / 230-Brass instrument / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"20-Trailer / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 40-Road / 64-Transport / 575-Traffic / 1020-Emergency vehicle / \n",
"293-Slide show / 444-Family / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"0-Games / 59-Winter sport / 81-Athlete / 112-Ice skating / 142-Hockey / 312-Ice rink / 2730-NHL 13 / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / \n",
"86-Plant / 136-Gardening / 279-Garden / 807-Seed / \n",
"0-Games / 2-Video game / 19-PC game / 47-Personal computer / \n",
"31-Disc jockey / 861-Mixing console / 3024-Numark Industries / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 94-Dashcam / 212-Highway / 286-Bus / \n",
"445-Guitar Hero / 2180-Frets on Fire / \n",
"762-Angel / \n",
"12-Food / 88-Machine / \n",
"1-Vehicle / 22-Nature / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 152-Track / 213-Forest / \n",
"1-Vehicle / 27-Motorcycle / 40-Road / 105-Motorcycling / 336-Scooter / \n",
"113-House / 310-The Sims / 328-Simulation video game / 564-The Sims 3 / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 74-Truck / 94-Dashcam / 212-Highway / \n",
"123-Naruto / \n",
"6-Animation / 15-Cartoon / 72-iPhone / 101-Telephone / \n",
"0-Games / 2-Video game / 8-Football / 90-Sports game / 180-FIFA 15 / \n",
"3-Concert / 5-Dance / 16-Performance art / 68-Lighting / \n",
"6-Animation / 15-Cartoon / 25-Toy / 140-LEGO / 359-Harry Potter (Literary Series) / \n",
"3-Concert / 7-Musician / 30-Drummer / 133-Nightclub / \n",
"1970-Axe / \n",
"619-Manufacturing / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"0-Games / 90-Sports game / 176-PlayStation 4 / 453-Pro Evolution Soccer / 1719-Pro Evolution Soccer 2015 / \n",
"56-Hair / 340-Afro-textured hair / \n",
"9-Music video / \n",
"1-Vehicle / 23-Mobile phone / 29-Smartphone / 101-Telephone / \n",
"1-Vehicle / 74-Truck / 86-Plant / 137-Tree / \n",
"0-Games / 269-Comic book / 1244-Trunks / \n",
"5-Dance / 49-School / 191-Cheerleading / 776-Kindergarten / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / 278-GoPro / 522-Helmet / 1323-Helmet camera / \n",
"25-Toy / 200-Doll / 412-Monster / 503-Monster High / 4332-Mother 3 / \n",
"1-Vehicle / 4-Car / 74-Truck / 659-Van / 2737-GAZ / \n",
"3-Concert / 7-Musician / \n",
"6-Animation / 313-Book / \n",
"25-Toy / 200-Doll / 483-Costume / \n",
"3-Concert / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"3-Concert / \n",
"0-Games / 256-Tennis / \n",
"261-Tool / 1226-Metalworking / \n",
"3-Concert / 68-Lighting / 263-Light / \n",
"9-Music video / \n",
"1532-Military band / \n",
"30-Drummer / 39-Drums / 44-Drums / \n",
"61-Art / \n",
"211-RuneScape / \n",
"12-Food / 26-Cooking / 32-Recipe / 236-Baking / 239-Roasting / 274-Meat / 560-Sugar / 1596-Cinnamon / 2042-Nut / \n",
"0-Games / 2-Video game / 1911-StepMania / 1990-Pac-Man / \n",
"6-Animation / 15-Cartoon / \n",
"93-Comedy / 125-Television / \n",
"12-Food / 26-Cooking / 32-Recipe / 109-Cooking show / 215-Eating / 808-Breakfast / \n",
"131-Computer / 513-Computer keyboard / \n",
"1-Vehicle / 4-Car / 449-Honda / 531-Hatchback / 2900-Honda Fit / \n",
"23-Mobile phone / 29-Smartphone / 125-Television / 1565-Logitech / \n",
"6-Animation / 15-Cartoon / \n",
"39-Drums / 44-Drums / 687-Taiko no Tatsujin / \n",
"6-Animation / 15-Cartoon / 116-Pokémon / 186-Pokémon / 1005-Pokémon Omega Ruby and Alpha Sapphire / 1153-Pokémon HeartGold and SoulSilver / 4379-Team Rocket / \n",
"25-Toy / 109-Cooking show / 447-Sand / \n",
"23-Mobile phone / 29-Smartphone / 125-Television / \n",
"10-Animal / 166-Cat / 346-Kitten / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 141-Microsoft Windows / 932-Microsoft Lumia / \n",
"12-Food / 88-Machine / 625-Factory / 812-Plastic / 2031-Molding / \n",
"1-Vehicle / 286-Bus / \n",
"5-Dance / \n",
"8-Football / \n",
"0-Games / 88-Machine / 366-Dragon / 535-Slot machine / \n",
"0-Games / 1083-Role-playing game / \n",
"275-Diving / 287-Human swimming / \n",
"5-Dance / 16-Performance art / \n",
"1-Vehicle / 4-Car / 12-Food / 25-Toy / 215-Eating / 475-Cheese / 3822-Mold / \n",
"9-Music video / \n",
"18-Outdoor recreation / 59-Winter sport / 112-Ice skating / 316-Figure skating / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"269-Comic book / \n",
"448-Planet / 1508-Spacecraft / 3413-Pluto / \n",
"6-Animation / 15-Cartoon / \n",
"1398-Triangle / \n",
"6-Animation / 15-Cartoon / 290-Pony / 343-My Little Pony / \n",
"86-Plant / 136-Gardening / 137-Tree / \n",
"0-Games / 2-Video game / 596-Monster Hunter / 2260-Monster Hunter Portable 3rd / \n",
"28-Fashion / 56-Hair / \n",
"1902-Flyff / \n",
"1612-Multi Theft Auto / \n",
"3-Concert / 38-Orchestra / \n",
"113-House / 279-Garden / 363-Soil / \n",
"341-Sketch comedy / 364-Sitcom / \n",
"42-Fishing / 275-Diving / 288-Underwater / 381-Underwater diving / 407-Scuba diving / \n",
"12-Food / 227-Water / 3639-Heat pump / \n",
"5-Dance / 78-Wedding / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 247-Stallion / 289-Dressage / \n",
"1-Vehicle / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 266-Helicopter / 392-Radio-controlled helicopter / 601-Shark / \n",
"0-Games / 59-Winter sport / 112-Ice skating / 142-Hockey / \n",
"0-Games / 8-Football / 13-Musical ensemble / 21-Stadium / 77-Arena / 79-American football / 203-Marching band / \n",
"3837-Kyūdō / \n",
"12-Food / 26-Cooking / 32-Recipe / 367-Drink / 867-Tea / \n",
"3-Concert / \n",
"325-Vampire / \n",
"6-Animation / 15-Cartoon / \n",
"3-Concert / 16-Performance art / \n",
"12-Food / 341-Sketch comedy / \n",
"5-Dance / 430-Ballroom dance / 1231-Mandolin / \n",
"10-Animal / 80-Horse / 247-Stallion / 289-Dressage / 2553-Colt / \n",
"209-University / 319-Robot / \n",
"1-Vehicle / 50-Aircraft / 266-Helicopter / 758-Air force / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / \n",
"36-Piano / 53-Keyboard / \n",
"5-Dance / 49-School / 209-University / \n",
"7-Musician / 36-Piano / 46-Choir / \n",
"1-Vehicle / 4-Car / 245-BMW / 3058-BMW 7 Series / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / 154-Wheel / \n",
"56-Hair / 367-Drink / \n",
"45-Cosmetics / 56-Hair / \n",
"7-Musician / 13-Musical ensemble / 203-Marching band / \n",
"5-Dance / \n",
"0-Games / 19-PC game / 35-Minecraft / \n",
"0-Games / 2-Video game / 8-Football / 21-Stadium / 54-Highlight film / 79-American football / 90-Sports game / 561-Madden NFL / \n",
"9-Music video / 31-Disc jockey / \n",
"0-Games / 8-Football / 90-Sports game / 180-FIFA 15 / 480-FIFA 13 / 848-FIFA 12 / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / \n",
"9-Music video / \n",
"15-Cartoon / 61-Art / 97-Drawing / 148-Painting / 408-Sketch / 528-Pencil / 628-Portrait / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 47-Personal computer / 1749-Titanfall / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 685-Luigi / 1170-Super Mario World / \n",
"0-Games / 1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 172-Motocross / 2930-MX vs. ATV / \n",
"67-Cycling / 69-Bicycle / 377-BMX bike / \n",
"22-Nature / 225-Lake / 2280-Cottage / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 165-League of Legends / \n",
"8-Football / 73-Ball / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"15-Cartoon / 185-Dragon Ball / 244-Goku / \n",
"10-Animal / 12-Food / 26-Cooking / 1514-Crab / \n",
"0-Games / 2-Video game / 474-Destiny / \n",
"10-Animal / 80-Horse / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 76-Ball / \n",
"3-Concert / 68-Lighting / 99-Christmas / 263-Light / 729-Christmas decoration / 1098-Christmas lights / \n",
"0-Games / 10-Animal / 80-Horse / 1871-Foal / \n",
"372-Playing card / 762-Angel / 1191-Tarot / \n",
"20-Trailer / 264-Star Wars / 1791-Stargate / \n",
"0-Games / 51-Strategy video game / 165-League of Legends / \n",
"0-Games / 1-Vehicle / 2-Video game / 140-LEGO / 2276-Lego Marvel Super Heroes / 4016-Lego City Undercover / \n",
"0-Games / 23-Mobile phone / 29-Smartphone / 72-iPhone / 4567-Cut the Rope / \n",
"1-Vehicle / 4-Car / 50-Aircraft / 1554-Range Rover / 3890-Range Rover Sport / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / \n",
"1-Vehicle / 4-Car / 349-Chevrolet / 406-Chevrolet / 659-Van / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 61-Art / 732-StarCraft / \n",
"46-Choir / \n",
"9-Music video / 22-Nature / 243-Sonic the Hedgehog / \n",
"5-Dance / 13-Musical ensemble / \n",
"10-Animal / 71-Dog / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / 115-Ballet / \n",
"1-Vehicle / 4-Car / 41-Sports car / 67-Cycling / 69-Bicycle / \n",
"6-Animation / 15-Cartoon / 61-Art / 97-Drawing / \n",
"49-School / 102-Building / \n",
"75-Wrestling / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 105-Motorcycling / \n",
"49-School / \n",
"5-Dance / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 38-Orchestra / \n",
"322-Mixtape / \n",
"61-Art / 479-Graffiti / \n",
"6-Animation / 15-Cartoon / 61-Art / 97-Drawing / 148-Painting / \n",
"330-Money / 458-Gold / 731-Coin / 912-Silver / 2095-Cash / 4247-Bullion / \n",
"3954-Camera Operator / \n",
"20-Trailer / \n",
"20-Trailer / \n",
"1-Vehicle / 4-Car / 11-Motorsport / \n",
"18-Outdoor recreation / 84-Snow / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / \n",
"0-Games / 6-Animation / 15-Cartoon / \n",
"66-Bollywood / \n",
"0-Games / 2-Video game / 601-Shark / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 210-Cookware and bakeware / \n",
"3-Concert / 68-Lighting / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 47-Personal computer / 145-Tablet computer / \n",
"5-Dance / 892-Aerobics / \n",
"12-Food / 52-Dish / 58-Cuisine / 215-Eating / 301-Meal / 1227-Sushi / \n",
"2207-Kemenche / \n",
"36-Piano / 53-Keyboard / 265-Electronic keyboard / \n",
"28-Fashion / 78-Wedding / 121-Photography / 190-Bride / 421-Gown / 2269-Wedding photography / 2873-Kimono / \n",
"8-Football / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / \n",
"0-Games / 535-Slot machine / 3952-Amusement arcade / \n",
"1-Vehicle / 4-Car / 277-Sport utility vehicle / 901-Mitsubishi / 3896-Mitsubishi Outlander / \n",
"0-Games / 2-Video game / 948-Far Cry / 1683-Far Cry 4 / \n",
"25-Toy / 179-Kitchen / 200-Doll / 439-Barbie / \n",
"643-Hero / \n",
"2207-Kemenche / \n",
"1060-Raw food / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 89-Comics / 185-Dragon Ball / 244-Goku / 869-Dragon Ball Z: Budokai Tenkaichi / 1165-Majin Buu / \n",
"148-Painting / \n",
"7-Musician / 13-Musical ensemble / 78-Wedding / 150-Violin / 401-Cello / 450-Viola / \n",
"345-Loudspeaker / 484-Subwoofer / \n",
"242-Paper / 370-Origami / 3300-Napkin / 4124-Napkin folding / \n",
"9-Music video / 99-Christmas / \n",
"28-Fashion / 45-Cosmetics / 56-Hair / 1811-Forever 21 / \n",
"5-Dance / \n",
"10-Animal / 91-Fish / 258-Aquarium / 1368-Gallon / \n",
"36-Piano / 53-Keyboard / 1492-Persona / 2896-Shin Megami Tensei: Persona 3 / \n",
"12-Food / 26-Cooking / 32-Recipe / 232-Cake / 236-Baking / 1081-Lemon / 1114-Stuffing / 1322-Biscuit / 2593-Sponge cake / \n",
"341-Sketch comedy / \n",
"8-Football / \n",
"183-Manga / 4038-Waistcoat / \n",
"130-Gymnastics / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 134-Call of Duty: Black Ops / 216-Call of Duty: Modern Warfare 3 / 704-Call of Duty: World at War / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 850-Left 4 Dead / 963-Left 4 Dead 2 / \n",
"0-Games / 6-Animation / 269-Comic book / 460-Yu-Gi-Oh! Trading Card Game / 494-Card game / \n",
"1-Vehicle / 4-Car / 114-Engine / 1521-Jaguar Cars / \n",
"20-Trailer / \n",
"1-Vehicle / 2-Video game / 20-Trailer / 124-Tractor / 174-Farm / 715-Farming Simulator / \n",
"10-Animal / 25-Toy / 48-Pet / 71-Dog / 182-Puppy / \n",
"5-Dance / 133-Nightclub / \n",
"6-Animation / 15-Cartoon / \n",
"9-Music video / \n",
"222-Surfing / \n",
"2840-Engraving / \n",
"282-Skin / 4665-Day spa / \n",
"1-Vehicle / 4-Car / 41-Sports car / 231-Supercar / 983-Lexus / 4158-Lexus LFA / \n",
"6-Animation / 116-Pokémon / \n",
"1-Vehicle / 62-Train / 103-Rail transport / 119-Locomotive / \n",
"18-Outdoor recreation / 22-Nature / 137-Tree / 213-Forest / 217-Hunting / \n",
"1-Vehicle / 27-Motorcycle / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 161-Train station / 224-Rapid transit / 1547-Passenger / \n",
"0-Games / \n",
"1-Vehicle / 27-Motorcycle / 336-Scooter / \n",
"6-Animation / 15-Cartoon / 3877-Ghost in the Shell / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 135-Drifting / 154-Wheel / 158-Tire / 493-Rim / \n",
"0-Games / 2-Video game / 89-Comics / 123-Naruto / 2771-Naruto: Clash of Ninja / \n",
"0-Games / 2-Video game / 116-Pokémon / 186-Pokémon / 1204-Pokémon Ruby and Sapphire / \n",
"425-City / \n",
"1-Vehicle / 31-Disc jockey / 74-Truck / \n",
"0-Games / 6-Animation / 15-Cartoon / 25-Toy / 1154-Anpanman / \n",
"36-Piano / 150-Violin / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / \n",
"75-Wrestling / 269-Comic book / \n",
"3-Concert / \n",
"0-Games / 23-Mobile phone / 29-Smartphone / 72-iPhone / 189-iPad / 272-iPod / \n",
"13-Musical ensemble / \n",
"298-Television advertisement / 337-Advertising / 3649-Metroid Prime / \n",
"5-Dance / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 37-Gadget / 55-Video game console / 292-Handheld game console / \n",
"31-Disc jockey / \n",
"2-Video game / 297-Grand Theft Auto IV / \n",
"6-Animation / 9-Music video / 15-Cartoon / \n",
"8-Football / \n",
"0-Games / 2-Video game / 328-Simulation video game / 2088-City-building game / 2307-SimCity / 3290-SimCity / \n",
"3-Concert / 16-Performance art / \n",
"12-Food / 26-Cooking / 32-Recipe / 226-Dessert / 295-Chocolate / 2386-Chocolate brownie / \n",
"131-Computer / 155-Camera / 278-GoPro / 633-Photograph / \n",
"1-Vehicle / 17-Racing / 18-Outdoor recreation / 59-Winter sport / 84-Snow / 177-Skiing / 789-Alpine skiing / \n",
"10-Animal / 166-Cat / 440-Lion / 537-Zoo / 3986-Cougar / \n",
"6-Animation / 352-Need for Speed / 2126-Need for Speed: Underground / \n",
"130-Gymnastics / \n",
"0-Games / 1138-Tower of Saviors / \n",
"9-Music video / \n",
"14-Guitar / 841-Butter / \n",
"0-Games / 2-Video game / 432-Kingdom Hearts / 665-Kingdom Hearts / 836-Sora / 941-Kingdom Hearts II / \n",
"0-Games / 2-Video game / 686-Diablo III / \n",
"0-Games / 2-Video game / \n",
"1-Vehicle / 4-Car / 74-Truck / \n",
"9-Music video / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"56-Hair / 106-Hairstyle / 607-Lace / \n",
"574-Elevator / \n",
"3-Concert / 7-Musician / 14-Guitar / 24-String instrument / \n",
"85-Combat / 130-Gymnastics / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 570-Road bicycle racing / \n",
"45-Cosmetics / \n",
"75-Wrestling / 85-Combat / \n",
"82-Airplane / 778-Gliding / 1263-Glider / \n",
"3-Concert / 16-Performance art / \n",
"0-Games / 8-Football / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 178-Samsung Galaxy / 1207-Samsung Galaxy S5 / \n",
"1-Vehicle / 4-Car / 531-Hatchback / 683-Renault / \n",
"5-Dance / 16-Performance art / \n",
"22-Nature / 223-Weather / 490-Rain / 499-Cloud / 502-Sky / \n",
"86-Plant / 136-Gardening / 261-Tool / 279-Garden / 585-Lawn / 2437-String trimmer / \n",
"5-Dance / 1117-Camcorder / \n",
"237-Final Fantasy / \n",
"325-Vampire / \n",
"9-Music video / \n",
"259-Fire / 282-Skin / \n",
"10-Animal / 132-Bird / 217-Hunting / 1373-Torero / \n",
"536-Glass / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"35-Minecraft / \n",
"504-Web page / \n",
"12-Food / 88-Machine / 533-Oil / 1769-Olive / 3027-Olive oil / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / \n",
"1-Vehicle / \n",
"3-Concert / 13-Musical ensemble / 16-Performance art / \n",
"9-Music video / \n",
"9-Music video / \n",
"0-Games / 791-Magic: The Gathering / \n",
"3-Concert / 5-Dance / 98-Festival / \n",
"3-Concert / 46-Choir / \n",
"36-Piano / 53-Keyboard / 104-Pianist / \n",
"66-Bollywood / \n",
"0-Games / 2-Video game / 229-Halo / 866-Halo 4 / \n",
"394-Map / \n",
"35-Minecraft / \n",
"110-Album / \n",
"5-Dance / \n",
"2-Video game / 43-Call of Duty / 134-Call of Duty: Black Ops / 635-Achievement / 924-Trophy / \n",
"44-Drums / \n",
"46-Choir / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"0-Games / 8-Football / 21-Stadium / 77-Arena / \n",
"385-Face / 2062-Kidney / \n",
"3-Concert / 68-Lighting / \n",
"88-Machine / 539-Home appliance / 733-Washing / 818-Washing machine / 1142-Laundry / 1175-Soap / 1456-Clothes dryer / 4325-Detergent / \n",
"0-Games / 2-Video game / 55-Video game console / 581-PlayStation / \n",
"9-Music video / 14-Guitar / 63-Acoustic guitar / 465-Microphone / \n",
"16-Performance art / 350-Circus / 648-Clown / \n",
"0-Games / 1112-Candy Crush Saga / \n",
"2222-Crocodile / \n",
"0-Games / 1-Vehicle / 2-Video game / 19-PC game / 33-Weapon / 96-Soldier / 192-Battlefield / 383-Battlefield 4 / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 1278-Labrador Retriever / 1377-Golden Retriever / \n",
"0-Games / 75-Wrestling / 85-Combat / \n",
"0-Games / 2-Video game / 6-Animation / 126-The Walt Disney Company / 1077-Lightning McQueen / 3484-Cars 2 / \n",
"376-Flute / \n",
"3889-Periodic Table / \n",
"3-Concert / 16-Performance art / \n",
"364-Sitcom / \n",
"6-Animation / 15-Cartoon / 61-Art / 97-Drawing / 343-My Little Pony / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / \n",
"452-Watch / \n",
"33-Weapon / 521-Metal / \n",
"6-Animation / 15-Cartoon / 801-Inuyasha / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 966-Sandwich / 1843-Ham / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 138-Call of Duty: Black Ops II / 216-Call of Duty: Modern Warfare 3 / \n",
"72-iPhone / 131-Computer / 504-Web page / \n",
"222-Surfing / 815-Surfboard / \n",
"20-Trailer / 43-Call of Duty / 422-Call of Duty: Ghosts / \n",
"10-Animal / 17-Racing / 80-Horse / 384-Horse racing / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 659-Van / 4453-Kei car / \n",
"6-Animation / 15-Cartoon / 4221-Saiyuki / \n",
"0-Games / 2-Video game / 6-Animation / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 169-Rallying / \n",
"75-Wrestling / 85-Combat / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 157-Jet aircraft / 388-Fighter aircraft / 758-Air force / 1483-United States Air Force / \n",
"1-Vehicle / 18-Outdoor recreation / 50-Aircraft / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 266-Helicopter / 392-Radio-controlled helicopter / 1126-Tyrannosaurus / \n",
"10-Animal / 523-Cattle / 534-Milk / \n",
"3-Concert / \n",
"3-Concert / \n",
"5-Dance / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"0-Games / 2-Video game / 23-Mobile phone / 29-Smartphone / 72-iPhone / 189-iPad / \n",
"6-Animation / \n",
"2-Video game / 121-Photography / 155-Camera / 276-Room / \n",
"66-Bollywood / 78-Wedding / 466-Gift / \n",
"68-Lighting / 286-Bus / \n",
"163-Running / 769-Red Bull / \n",
"31-Disc jockey / \n",
"1-Vehicle / 4-Car / 1074-Hyundai / \n",
"122-Weight training / 529-Squat / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / 1019-Christ / \n",
"1-Vehicle / 4-Car / 493-Rim / \n",
"9-Music video / 66-Bollywood / \n",
"25-Toy / 412-Monster / 713-Egg / 1047-Easter egg / \n",
"1151-Temple / 1353-Bishop / \n",
"31-Disc jockey / \n",
"0-Games / 599-Dofus / \n",
"6-Animation / 15-Cartoon / \n",
"122-Weight training / \n",
"341-Sketch comedy / \n",
"0-Games / 2-Video game / 33-Weapon / 2436-Zeus / \n",
"1-Vehicle / 62-Train / 119-Locomotive / 2369-RailWorks / \n",
"121-Photography / 293-Slide show / \n",
"55-Video game console / 127-Xbox 360 / 181-Xbox / 386-Xbox / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 76-Ball / \n",
"1-Vehicle / 4-Car / 158-Tire / 403-Toyota / 659-Van / 1624-Minivan / \n",
"7-Musician / 2641-Radha / \n",
"448-Planet / 4349-Neptune / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 44-Drums / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 2163-Rechargeable battery / \n",
"23-Mobile phone / 29-Smartphone / 178-Samsung Galaxy / 1776-Samsung Galaxy Note II / \n",
"83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"60-Basketball / \n",
"3-Concert / 7-Musician / \n",
"1-Vehicle / 18-Outdoor recreation / 42-Fishing / 87-Boat / 225-Lake / 268-Recreational fishing / 896-Largemouth bass / \n",
"10-Animal / 80-Horse / 247-Stallion / 404-Livestock / 438-Mare / 2972-Andalusian horse / \n",
"0-Games / 6-Animation / 15-Cartoon / 25-Toy / 82-Airplane / 125-Television / 323-Toddler / \n",
"47-Personal computer / 145-Tablet computer / 1802-Jubeat / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 184-Vegetable / 236-Baking / 454-Sauce / 589-Grilling / 1186-Seafood / \n",
"8-Football / \n",
"5-Dance / 49-School / \n",
"36-Piano / 53-Keyboard / 265-Electronic keyboard / \n",
"3-Concert / \n",
"10-Animal / 25-Toy / 166-Cat / 346-Kitten / 1234-Mouse / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"25-Toy / \n",
"20-Trailer / 2075-Titan / \n",
"5-Dance / 66-Bollywood / \n",
"7-Musician / 9-Music video / \n",
"1-Vehicle / 4-Car / 86-Plant / 124-Tractor / 156-Agriculture / 174-Farm / 249-Heavy equipment / 332-Mercedes-Benz / 459-Plough / 660-Rural area / \n",
"9-Music video / \n",
"0-Games / 8-Football / 54-Highlight film / \n",
"0-Games / 1-Vehicle / 4-Car / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / 2402-Grand Theft Auto / \n",
"20-Trailer / \n",
"593-Moon / \n",
"9-Music video / \n",
"110-Album / \n",
"12-Food / 2371-Hot dog / \n",
"0-Games / 505-Mega Man / 1664-Zero / 3035-Mega Man Zero / \n",
"1862-Corel / \n",
"36-Piano / 53-Keyboard / 246-Furniture / 457-Knitting / 546-Stitch / 1479-Quilt / 2966-Patchwork / \n",
"47-Personal computer / 145-Tablet computer / \n",
"0-Games / 51-Strategy video game / 165-League of Legends / \n",
"1151-Temple / \n",
"6-Animation / 15-Cartoon / \n",
"42-Fishing / 1836-Fly tying / \n",
"1-Vehicle / 4-Car / 64-Transport / 74-Truck / 158-Tire / 249-Heavy equipment / 1772-Forklift / \n",
"61-Art / 97-Drawing / 148-Painting / 303-Paint / 408-Sketch / 528-Pencil / \n",
"47-Personal computer / 145-Tablet computer / 155-Camera / 802-Google Nexus / 2264-Nexus 7 / \n",
"2060-Roulette / \n",
"9-Music video / 147-River / \n",
"3412-Daredevil / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 2059-Doberman Pinscher / \n",
"3-Concert / 68-Lighting / \n",
"20-Trailer / 66-Bollywood / \n",
"110-Album / \n",
"0-Games / 2-Video game / 8-Football / 90-Sports game / 453-Pro Evolution Soccer / 3526-Pro Evolution Soccer 2009 / \n",
"55-Video game console / 127-Xbox 360 / 181-Xbox / 386-Xbox / \n",
"12-Food / \n",
"117-Boxing / \n",
"1-Vehicle / 4-Car / 50-Aircraft / 82-Airplane / 157-Jet aircraft / 231-Supercar / 388-Fighter aircraft / 569-Lamborghini / 2510-McDonnell Douglas F/A-18 Hornet / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 74-Truck / 349-Chevrolet / 406-Chevrolet / 4251-Chevrolet K5 Blazer / \n",
"0-Games / 51-Strategy video game / \n",
"67-Cycling / 69-Bicycle / \n",
"56-Hair / \n",
"6-Animation / 15-Cartoon / 1563-Stick figure / \n",
"333-Nike; Inc. / \n",
"0-Games / 2-Video game / 8-Football / 90-Sports game / 480-FIFA 13 / \n",
"3-Concert / 68-Lighting / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 94-Dashcam / \n",
"7-Musician / 13-Musical ensemble / 21-Stadium / 54-Highlight film / 79-American football / 203-Marching band / 1675-Drum and bugle corps / \n",
"31-Disc jockey / \n",
"164-Beach / 222-Surfing / \n",
"0-Games / 2-Video game / 33-Weapon / 257-Counter-Strike / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 105-Motorcycling / 172-Motocross / \n",
"1016-Glove / \n",
"67-Cycling / 69-Bicycle / 570-Road bicycle racing / \n",
"1-Vehicle / 4-Car / 527-Battery / 574-Elevator / 1454-Stairs / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 229-Halo / 644-Halo: Reach / \n",
"7-Musician / 24-String instrument / 38-Orchestra / 150-Violin / 401-Cello / 450-Viola / 545-Quartet (ensemble) / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / \n",
"25-Toy / 242-Paper / 1094-Puppet / \n",
"6-Animation / 123-Naruto / 285-Sasuke Uchiha / 622-Sakura Haruno / \n",
"0-Games / 2-Video game / 2143-18 Wheels of Steel / \n",
"110-Album / \n",
"1198-Birth / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 257-Counter-Strike / \n",
"3-Concert / 5-Dance / 49-School / \n",
"5-Dance / 95-Talent show / \n",
"13-Musical ensemble / \n",
"6-Animation / 15-Cartoon / 538-Illustration / \n",
"97-Drawing / 882-Pen / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 150-Violin / 639-Fiddle / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"78-Wedding / 121-Photography / 190-Bride / 421-Gown / 2269-Wedding photography / \n",
"12-Food / 26-Cooking / 1408-Snack / \n",
"12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / 437-Cream / 684-Ice cream / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 184-Vegetable / 210-Cookware and bakeware / 239-Roasting / 542-Potato / 597-Soup / 2405-Zucchini / 3113-Multicooker / \n",
"1-Vehicle / 4-Car / 25-Toy / 92-Radio-controlled model / 140-LEGO / 202-Radio-controlled car / 319-Robot / \n",
"12-Food / 1227-Sushi / \n",
"102-Building / 113-House / 170-Home improvement / 433-Construction / 707-Roof / \n",
"0-Games / 2-Video game / 1580-Crysis 2 / \n",
"8-Football / \n",
"0-Games / 6-Animation / 116-Pokémon / 186-Pokémon / 690-Pokémon X and Y / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 2519-Grand Theft Auto: Liberty City Stories / \n",
"31-Disc jockey / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 84-Snow / 169-Rallying / \n",
"78-Wedding / \n",
"0-Games / 2-Video game / 34-Action-adventure game / \n",
"28-Fashion / 305-Ski / \n",
"9-Music video / \n",
"766-President of the United States / \n",
"2-Video game / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / \n",
"66-Bollywood / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"5-Dance / \n",
"6-Animation / 10-Animal / 15-Cartoon / 290-Pony / 343-My Little Pony / 1164-Cartoon Network / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"124-Tractor / 156-Agriculture / 174-Farm / 741-Mower / 1037-Grass / \n",
"78-Wedding / 121-Photography / 293-Slide show / 633-Photograph / \n",
"6-Animation / 15-Cartoon / 2624-Lelouch Lamperouge / \n",
"106-Hairstyle / 1359-Hair iron / 3178-Clothes iron / \n",
"75-Wrestling / 1080-WWE 2K15 / \n",
"10-Animal / 80-Horse / 438-Mare / \n",
"33-Weapon / 315-Knife / 434-Ninja / \n",
"47-Personal computer / 131-Computer / 309-Computer hardware / 602-Central processing unit / \n",
"45-Cosmetics / \n",
"9-Music video / \n",
"8-Football / 595-Champion / \n",
"89-Comics / 1469-X-Men / \n",
"122-Weight training / 204-Gym / 661-Stretching / \n",
"2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 155-Camera / 178-Samsung Galaxy / 487-Samsung Electronics / 883-Samsung Galaxy S4 / \n",
"6-Animation / 15-Cartoon / 20-Trailer / 298-Television advertisement / \n",
"6-Animation / 15-Cartoon / 679-Mickey Mouse / 1234-Mouse / 1445-Donald Duck / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / 265-Electronic keyboard / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 169-Rallying / 299-Volkswagen Passenger Cars / \n",
"512-Printing / 1757-Screen printing / \n",
"1-Vehicle / 18-Outdoor recreation / 147-River / 1018-Salmon / 1195-Rafting / \n",
"3369-Diego de la Vega / \n",
"10-Animal / \n",
"36-Piano / 88-Machine / 457-Knitting / 476-Textile / 1100-Hat / \n",
"22-Nature / 223-Weather / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 210-Cookware and bakeware / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / 1103-Gibson Les Paul / \n",
"10-Animal / 25-Toy / 71-Dog / 524-Littlest Pet Shop / \n",
"5-Dance / 1726-Harlem Shake (meme) / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 184-Vegetable / \n",
"0-Games / 2-Video game / 43-Call of Duty / 118-Grand Theft Auto V / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 101-Telephone / 554-iPhone 4 / 1366-LG Optimus series / \n",
"4089-Maracas / \n",
"45-Cosmetics / 121-Photography / 155-Camera / \n",
"15-Cartoon / 185-Dragon Ball / 244-Goku / 418-Gohan / \n",
"0-Games / 2-Video game / 6-Animation / 51-Strategy video game / 382-Dota 2 / 549-Defense of the Ancients / \n",
"25-Toy / 417-Play-Doh / 587-Clay / 1185-Plasticine / \n",
"5-Dance / \n",
"22-Nature / 694-Tai chi / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 47-Personal computer / 297-Grand Theft Auto IV / 658-Grand Theft Auto: The Lost and Damned / 1923-Niko Bellic / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 55-Video game console / 320-Super Smash Bros. / 4449-King Dedede / \n",
"1-Vehicle / 4-Car / 484-Subwoofer / \n",
"9-Music video / 147-River / \n",
"13-Musical ensemble / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / 44-Drums / 3066-Timbales / \n",
"0-Games / 2-Video game / 739-Guild Wars / 839-Guild Wars 2 / \n",
"12-Food / 226-Dessert / 232-Cake / 578-Cookie / 759-Icing / 3641-Royal icing / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 134-Call of Duty: Black Ops / 713-Egg / 1047-Easter egg / 1069-Easter egg / \n",
"0-Games / 2-Video game / 6-Animation / 2450-Inazuma Eleven / 3426-Inazuma Eleven GO / \n",
"0-Games / 8-Football / 21-Stadium / 73-Ball / 90-Sports game / \n",
"3-Concert / 16-Performance art / 68-Lighting / \n",
"1-Vehicle / 124-Tractor / 136-Gardening / 156-Agriculture / 363-Soil / 459-Plough / 2079-Two-wheel tractor / \n",
"24-String instrument / 642-Leaf / 657-Autumn / 842-Harp / \n",
"33-Weapon / 217-Hunting / 2396-Coyote / \n",
"3-Concert / 7-Musician / 14-Guitar / 30-Drummer / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / 490-Rain / 878-Composer / \n",
"1-Vehicle / 4-Car / 1237-Baby carriage / 1932-Child safety seat / \n",
"88-Machine / 261-Tool / 400-Woodturning / 521-Metal / 972-Lathe / \n",
"5-Dance / 16-Performance art / \n",
"121-Photography / 155-Camera / 278-GoPro / \n",
"1-Vehicle / 4-Car / 114-Engine / 124-Tractor / \n",
"3-Concert / 5-Dance / 16-Performance art / 191-Cheerleading / \n",
"61-Art / 479-Graffiti / \n",
"122-Weight training / \n",
"14-Guitar / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 230-Brass instrument / 427-Trumpet / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 1125-God of War / 1432-Kratos / 1743-God of War / 2103-God of War III / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 105-Motorcycling / \n",
"130-Gymnastics / 1255-Ribbon / \n",
"0-Games / 2-Video game / 257-Counter-Strike / 702-Counter-Strike: Source / 826-Source Engine / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 651-Stock car racing / \n",
"9-Music video / 14-Guitar / 23-Mobile phone / 29-Smartphone / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 2626-Shih Tzu / \n",
"414-Computer monitor / \n",
"447-Sand / 981-Sculpture / 3282-Sand art and play / \n",
"20-Trailer / \n",
"0-Games / 1071-Hearthstone: Heroes of Warcraft / \n",
"2-Video game / 47-Personal computer / 513-Computer keyboard / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 70-Driving / 451-Cockpit / 3353-Chevrolet Monte Carlo / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 428-Rail transport modelling / \n",
"8-Football / 54-Highlight film / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / 315-Knife / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"1-Vehicle / 94-Dashcam / \n",
"0-Games / 2-Video game / 19-PC game / 951-Teenage Mutant Ninja Turtles / \n",
"183-Manga / 2065-Tsubasa: Reservoir Chronicle / \n",
"278-GoPro / \n",
"46-Choir / \n",
"1-Vehicle / 4-Car / 61-Art / \n",
"154-Wheel / \n",
"2-Video game / 47-Personal computer / 228-Laptop / 2481-Cooler Master / \n",
"6-Animation / 15-Cartoon / 1866-Katara / \n",
"0-Games / 60-Basketball / 168-Basketball moves / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 378-Drag racing / \n",
"6-Animation / 701-Beyblade / 3868-Beyblade: Shogun Steel / \n",
"0-Games / 2-Video game / 116-Pokémon / 186-Pokémon / 376-Flute / \n",
"30-Drummer / 39-Drums / 44-Drums / \n",
"5-Dance / 46-Choir / 49-School / \n",
"12-Food / \n",
"5-Dance / \n",
"14-Guitar / 95-Talent show / \n",
"3-Concert / 7-Musician / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / 30-Drummer / 39-Drums / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 302-Call of Duty 4: Modern Warfare / \n",
"10-Animal / 42-Fishing / 91-Fish / 893-Fish as food / 1186-Seafood / 2125-Fillet / \n",
"75-Wrestling / \n",
"3-Concert / 7-Musician / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 114-Engine / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 184-Vegetable / 500-Rice / 572-Indian cuisine / 3598-Lime / \n",
"61-Art / 148-Painting / 936-Watercolor paint / \n",
"23-Mobile phone / 37-Gadget / 125-Television / 1565-Logitech / \n",
"6-Animation / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / 2241-Plectrum / \n",
"28-Fashion / 45-Cosmetics / 173-Eye shadow / 193-Eye liner / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 556-Skateboarding trick / 1315-Kickflip / \n",
"22-Nature / 376-Flute / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / 99-Christmas / \n",
"83-Skateboarding / 703-Roller skating / 1477-Aggressive inline skating / 1937-Inline skates / 2747-Inline skating / \n",
"7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"162-Medicine / \n",
"0-Games / 2-Video game / 35-Minecraft / 55-Video game console / 111-PlayStation 3 / 127-Xbox 360 / 176-PlayStation 4 / 181-Xbox / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 345-Loudspeaker / 484-Subwoofer / 618-Fiat Automobiles / 2865-Fiat Punto / \n",
"445-Guitar Hero / \n",
"0-Games / 2-Video game / 578-Cookie / 1170-Super Mario World / \n",
"10-Animal / 12-Food / 26-Cooking / 32-Recipe / 71-Dog / 232-Cake / \n",
"1-Vehicle / 27-Motorcycle / 67-Cycling / 69-Bicycle / 520-Bicycle frame / 625-Factory / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / 230-Brass instrument / 634-Trombone / \n",
"3175-Hide-and-seek / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 198-Off-road vehicle / 751-Desert / 975-Dune / \n",
"3556-Charlie Brown / \n",
"1088-Infomercial / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 275-Diving / 288-Underwater / 381-Underwater diving / 407-Scuba diving / \n",
"9-Music video / 78-Wedding / 190-Bride / \n",
"7-Musician / \n",
"4235-American Kenpo / \n",
"0-Games / 2-Video game / 47-Personal computer / 51-Strategy video game / \n",
"6-Animation / 33-Weapon / \n",
"7-Musician / 38-Orchestra / 150-Violin / \n",
"61-Art / 98-Festival / 479-Graffiti / \n",
"0-Games / 43-Call of Duty / 1078-Installation art / 2434-Call of Duty / \n",
"162-Medicine / 830-Bed / \n",
"0-Games / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / \n",
"2-Video game / \n",
"98-Festival / \n",
"0-Games / 2-Video game / 89-Comics / 185-Dragon Ball / 244-Goku / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / \n",
"15-Cartoon / 61-Art / 148-Painting / \n",
"20-Trailer / \n",
"296-Saxophone / 3199-Baritone saxophone / \n",
"47-Personal computer / 1125-God of War / \n",
"0-Games / 35-Minecraft / 102-Building / \n",
"86-Plant / 136-Gardening / 1403-Flower bouquet / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 55-Video game console / 320-Super Smash Bros. / 443-Wii U / 697-Super Smash Bros. for Nintendo 3DS and Wii U / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 38-Orchestra / \n",
"45-Cosmetics / 282-Skin / \n",
"0-Games / 462-Table / 489-Board game / 922-Tabletop game / \n",
"0-Games / 2-Video game / 55-Video game console / 578-Cookie / \n",
"1-Vehicle / 4-Car / 41-Sports car / 650-Convertible / 3979-Maserati GranTurismo / \n",
"8-Football / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / \n",
"1-Vehicle / 4-Car / \n",
"5-Dance / 95-Talent show / \n",
"238-Nail / 300-Nail art / 304-Nail polish / 324-Finger / 823-Artificial nails / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 19-PC game / 111-PlayStation 3 / \n",
"3-Concert / \n",
"1532-Military band / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"84-Snow / 120-Winter / 137-Tree / 578-Cookie / 795-Cupcake / \n",
"1-Vehicle / 39-Drums / 477-Transformers / 2835-Fallen / \n",
"3-Concert / 7-Musician / \n",
"1-Vehicle / 17-Racing / 67-Cycling / 69-Bicycle / 570-Road bicycle racing / \n",
"93-Comedy / 355-Comedian / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / 3223-Reindeer / \n",
"47-Personal computer / 228-Laptop / 309-Computer hardware / 513-Computer keyboard / \n",
"3-Concert / \n",
"419-Earth / 448-Planet / 712-Outer space / 1686-Orbit / \n",
"7-Musician / 9-Music video / 13-Musical ensemble / 14-Guitar / \n",
"3-Concert / \n",
"3837-Kyūdō / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 79-American football / 81-Athlete / \n",
"5-Dance / 16-Performance art / 78-Wedding / 190-Bride / \n",
"1-Vehicle / 4-Car / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"5-Dance / 9-Music video / 66-Bollywood / \n",
"3-Concert / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"1-Vehicle / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 393-Unmanned aerial vehicle / 1311-Amazon.com; Inc. / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 431-Call of Duty: Advanced Warfare / \n",
"325-Vampire / \n",
"31-Disc jockey / \n",
"3-Concert / 68-Lighting / \n",
"162-Medicine / 468-Biology / 1596-Cinnamon / \n",
"25-Toy / \n",
"20-Trailer / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / 465-Microphone / \n",
"3-Concert / 16-Performance art / \n",
"1-Vehicle / 4-Car / \n",
"3-Concert / \n",
"396-One Piece / \n",
"1-Vehicle / 4-Car / 74-Truck / 158-Tire / 349-Chevrolet / 375-Pickup truck / 406-Chevrolet / 1518-Chevrolet Silverado / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / 318-Church / \n",
"0-Games / 2-Video game / 35-Minecraft / 807-Seed / 810-Village / \n",
"0-Games / 2-Video game / 55-Video game console / 199-Wii / 1220-New Super Mario Bros / \n",
"22-Nature / \n",
"0-Games / 2-Video game / 6-Animation / 269-Comic book / 460-Yu-Gi-Oh! Trading Card Game / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / 116-Pokémon / 186-Pokémon / 1254-Cave / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"1737-Dance Dance Revolution / \n",
"0-Games / 8-Football / 76-Ball / \n",
"325-Vampire / \n",
"3-Concert / 7-Musician / \n",
"6-Animation / 15-Cartoon / 185-Dragon Ball / 244-Goku / 418-Gohan / 511-Mortal Kombat / 1304-Cell / \n",
"369-Resort / \n",
"1253-Ballet Dancer / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / 230-Brass instrument / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 57-Race track / 70-Driving / 94-Dashcam / \n",
"12-Food / 720-Wine / \n",
"3579-Mona Lisa / \n",
"3-Concert / 7-Musician / 24-String instrument / \n",
"1-Vehicle / 4-Car / 234-Ford / 1634-Ford Fiesta / \n",
"12-Food / 1576-Doughnut / \n",
"38-Orchestra / \n",
"0-Games / 6-Animation / \n",
"10-Animal / 86-Plant / 91-Fish / 136-Gardening / 258-Aquarium / \n",
"6-Animation / 15-Cartoon / 116-Pokémon / 1104-Ash Ketchum / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 70-Driving / 94-Dashcam / 515-Dodge / 647-Muscle car / \n",
"223-Weather / 419-Earth / 448-Planet / \n",
"49-School / 99-Christmas / 908-Video card / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 185-Dragon Ball / 199-Wii / 244-Goku / 518-Mario Kart / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 76-Ball / \n",
"3-Concert / 7-Musician / 14-Guitar / 30-Drummer / \n",
"66-Bollywood / \n",
"5-Dance / 46-Choir / 49-School / \n",
"3-Concert / 7-Musician / 16-Performance art / 38-Orchestra / \n",
"31-Disc jockey / \n",
"0-Games / 2-Video game / 19-PC game / 34-Action-adventure game / 380-Assassin's Creed / \n",
"768-Diamond / 2507-Macramé / 3063-Friendship bracelet / \n",
"3-Concert / \n",
"75-Wrestling / 1042-Jumbotron / \n",
"0-Games / 2-Video game / 79-American football / 90-Sports game / 330-Money / 561-Madden NFL / 1813-Madden NFL 13 / 2916-Madden NFL 12 / \n",
"365-Fireworks / \n",
"60-Basketball / \n",
"0-Games / 8-Football / 20-Trailer / 21-Stadium / 77-Arena / \n",
"1-Vehicle / 4-Car / 148-Painting / \n",
"12-Food / 539-Home appliance / 818-Washing machine / 1142-Laundry / 2838-Cotton / \n",
"5-Dance / 9-Music video / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 105-Motorcycling / 172-Motocross / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 1951-Sport bike / 2133-Road racing / \n",
"3-Concert / \n",
"0-Games / 328-Simulation video game / \n",
"308-Ship / 605-Cruise ship / 1402-Suite / \n",
"10-Animal / 48-Pet / 132-Bird / 342-Windows Media Video / 364-Sitcom / 598-Parrot / \n",
"9-Music video / \n",
"83-Skateboarding / 108-Skateboard / 1105-Fingerboard / \n",
"28-Fashion / 45-Cosmetics / 238-Nail / \n",
"3-Concert / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 67-Cycling / 69-Bicycle / 105-Motorcycling / 565-Mud bogging / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 70-Driving / 94-Dashcam / 212-Highway / 332-Mercedes-Benz / \n",
"68-Lighting / 113-House / 170-Home improvement / 353-Wall / 879-Ceiling / 3130-Loudspeaker enclosure / \n",
"3-Concert / 7-Musician / 39-Drums / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 79-American football / 81-Athlete / \n",
"0-Games / 51-Strategy video game / 269-Comic book / 791-Magic: The Gathering / \n",
"25-Toy / 524-Littlest Pet Shop / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 1247-Penalty kick / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 216-Call of Duty: Modern Warfare 3 / \n",
"522-Helmet / 1352-Airbrush / 4152-Aerography / \n",
"61-Art / 148-Painting / 2538-Painter / \n",
"83-Skateboarding / 108-Skateboard / 555-Longboard / 667-Longboarding / \n",
"6-Animation / 9-Music video / 15-Cartoon / 538-Illustration / \n",
"0-Games / 8-Football / 90-Sports game / 180-FIFA 15 / \n",
"12-Food / 26-Cooking / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 226-Dessert / 295-Chocolate / 1569-Steamed rice / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / \n",
"3-Concert / \n",
"3-Concert / 8-Football / 21-Stadium / \n",
"10-Animal / 80-Horse / 247-Stallion / 289-Dressage / 290-Pony / \n",
"347-Climbing / \n",
"1-Vehicle / 4-Car / \n",
"1-Vehicle / 50-Aircraft / 64-Transport / 82-Airplane / 139-Airport / 157-Jet aircraft / 175-Airliner / 188-Airline / 254-Runway / 719-Boeing 747 / 923-Boeing 777 / 1529-Jet engine / 2174-Delta Air Lines / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 92-Radio-controlled model / 147-River / 149-Model aircraft / 160-Radio-controlled aircraft / 225-Lake / \n",
"5-Dance / \n",
"28-Fashion / 56-Hair / 106-Hairstyle / 1576-Doughnut / 2663-Chignon / \n",
"661-Stretching / 920-Chair / \n",
"5-Dance / 49-School / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 73-Ball / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 728-Dirt track racing / \n",
"10-Animal / 80-Horse / 290-Pony / 404-Livestock / \n",
"1-Vehicle / 18-Outdoor recreation / 22-Nature / 67-Cycling / 69-Bicycle / 220-Mountain bike / 284-Mountain biking / \n",
"229-Halo / 866-Halo 4 / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / 150-Violin / 726-Clarinet / \n",
"538-Illustration / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / \n",
"28-Fashion / 45-Cosmetics / 99-Christmas / \n",
"1-Vehicle / 4-Car / 154-Wheel / 158-Tire / \n",
"0-Games / 19-PC game / 1071-Hearthstone: Heroes of Warcraft / \n",
"209-University / \n",
"45-Cosmetics / 173-Eye shadow / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"122-Weight training / 529-Squat / 4483-Burpee / \n",
"18-Outdoor recreation / 59-Winter sport / 267-Mountain / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 70-Driving / 94-Dashcam / 135-Drifting / 683-Renault / \n",
"248-Hotel / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"5-Dance / \n",
"1-Vehicle / 4-Car / 114-Engine / 3934-Radiator / \n",
"9-Music video / 911-Studio / \n",
"0-Games / 2-Video game / 19-PC game / 686-Diablo III / 872-Diablo / \n",
"1-Vehicle / 22-Nature / 124-Tractor / 136-Gardening / 156-Agriculture / 174-Farm / 459-Plough / 542-Potato / 868-Tram / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 172-Motocross / \n",
"242-Paper / 370-Origami / 466-Gift / \n",
"18-Outdoor recreation / 23-Mobile phone / 29-Smartphone / 72-iPhone / \n",
"2-Video game / 2815-F-Zero / \n",
"1-Vehicle / 25-Toy / 62-Train / 1279-Plarail / \n",
"3-Concert / \n",
"12-Food / 1739-Fireplace / \n",
"1-Vehicle / 10-Animal / 523-Cattle / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / \n",
"2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / 55-Video game console / 178-Samsung Galaxy / 581-PlayStation / 3361-Samsung Galaxy Y / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 145-Tablet computer / 178-Samsung Galaxy / \n",
"1-Vehicle / 4-Car / 114-Engine / 259-Fire / \n",
"10-Animal / 48-Pet / 723-Pig / \n",
"0-Games / 2-Video game / 55-Video game console / 292-Handheld game console / 320-Super Smash Bros. / 782-Super Smash Bros. Melee / \n",
"3-Concert / 5-Dance / \n",
"0-Games / 2-Video game / 55-Video game console / 2348-Chrono Trigger / \n",
"0-Games / 2-Video game / 685-Luigi / 3900-Mario & Luigi: Superstar Saga / \n",
"1955-Letter / \n",
"0-Games / 137-Tree / 174-Farm / 211-RuneScape / 4192-Oak / \n",
"0-Games / 2-Video game / 834-Combat Arms / \n",
"3-Concert / \n",
"6-Animation / 15-Cartoon / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / 224-Rapid transit / 691-Cargo / 821-Rail freight transport / 1904-Electric locomotive / \n",
"624-Number / \n",
"89-Comics / 2368-Marvel Universe / \n",
"0-Games / 372-Playing card / 494-Card game / \n",
"3-Concert / 5-Dance / 133-Nightclub / \n",
"10-Animal / 48-Pet / 537-Zoo / 571-Bear / 1160-Giant panda / \n",
"59-Winter sport / 112-Ice skating / 142-Hockey / \n",
"0-Games / 2-Video game / 19-PC game / 20-Trailer / 1232-Perfect World / 4467-Jade Dynasty / \n",
"0-Games / 326-Metin2 / \n",
"28-Fashion / 45-Cosmetics / 56-Hair / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 235-Lipstick / 291-Rouge / 436-Concealer / \n",
"75-Wrestling / \n",
"31-Disc jockey / \n",
"0-Games / 8-Football / 76-Ball / \n",
"110-Album / \n",
"10-Animal / 80-Horse / 247-Stallion / \n",
"3-Concert / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 19-PC game / 34-Action-adventure game / 118-Grand Theft Auto V / 127-Xbox 360 / 176-PlayStation 4 / 330-Money / \n",
"6-Animation / 10-Animal / 15-Cartoon / 126-The Walt Disney Company / 2089-Winnie-the-Pooh / \n",
"0-Games / 47-Personal computer / 201-Grand Theft Auto: San Andreas / \n",
"22-Nature / 144-Amusement park / 147-River / \n",
"12-Food / \n",
"6-Animation / 15-Cartoon / 1563-Stick figure / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"10-Animal / 12-Food / 26-Cooking / 52-Dish / \n",
"1-Vehicle / 4-Car / 70-Driving / 94-Dashcam / 2214-Multi-valve / 2989-Opel Vectra / \n",
"10-Animal / 17-Racing / 80-Horse / 247-Stallion / 384-Horse racing / \n",
"159-World of Warcraft / \n",
"45-Cosmetics / 173-Eye shadow / 195-Mascara / 196-Eye / 291-Rouge / 436-Concealer / \n",
"1-Vehicle / 87-Boat / 147-River / 225-Lake / 1358-Canyon / \n",
"468-Biology / \n",
"10-Animal / 588-Reptile / 1444-Lizard / 3060-Chameleon / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 151-Landing / 157-Jet aircraft / 187-Takeoff / 388-Fighter aircraft / 758-Air force / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / 2402-Grand Theft Auto / 3226-Chucky / \n",
"49-School / 410-Teacher / 466-Gift / 2914-Wreath / 4129-Crayon / \n",
"0-Games / 2-Video game / 199-Wii / 443-Wii U / \n",
"122-Weight training / 529-Squat / 661-Stretching / \n",
"3-Concert / 559-Night / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"0-Games / 51-Strategy video game / 165-League of Legends / \n",
"20-Trailer / \n",
"93-Comedy / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / \n",
"5-Dance / 16-Performance art / \n",
"1-Vehicle / 27-Motorcycle / 154-Wheel / 158-Tire / 336-Scooter / 1448-Vespa / \n",
"1-Vehicle / 27-Motorcycle / 3004-Softail / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / \n",
"1-Vehicle / 4-Car / \n",
"14-Guitar / \n",
"5-Dance / \n",
"5-Dance / \n",
"22-Nature / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 240-Arcade game / 1990-Pac-Man / \n",
"66-Bollywood / \n",
"9-Music video / \n",
"93-Comedy / \n",
"0-Games / 2-Video game / 4-Car / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"10-Animal / 48-Pet / 71-Dog / 132-Bird / 182-Puppy / 598-Parrot / 2626-Shih Tzu / \n",
"93-Comedy / \n",
"1-Vehicle / 4-Car / 332-Mercedes-Benz / 891-Mercedes-AMG / 3915-Mercedes-Benz CLS-Class / \n",
"0-Games / 2-Video game / 50-Aircraft / 55-Video game console / 2101-Ace Combat / \n",
"8-Football / \n",
"2-Video game / 23-Mobile phone / 29-Smartphone / 72-iPhone / 272-iPod / 331-iPod touch / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / 203-Marching band / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 157-Jet aircraft / 175-Airliner / 188-Airline / 254-Runway / 472-Boeing 737 / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"162-Medicine / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / 51-Strategy video game / 1120-Tower defense / \n",
"5-Dance / 16-Performance art / 115-Ballet / \n",
"12-Food / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"10-Animal / 48-Pet / 132-Bird / \n",
"5-Dance / 16-Performance art / 115-Ballet / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / \n",
"6-Animation / 15-Cartoon / 20-Trailer / \n",
"10-Animal / 71-Dog / \n",
"1-Vehicle / 4-Car / 94-Dashcam / 155-Camera / 559-Night / \n",
"809-Logo / \n",
"23-Mobile phone / 29-Smartphone / 3577-Walkman / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 90-Sports game / 180-FIFA 15 / 561-Madden NFL / \n",
"1143-Marimba / 1734-Xylophone / 2878-Vibraphone / \n",
"6-Animation / 15-Cartoon / 1741-Pineapple / \n",
"1266-Death Note / \n",
"79-American football / \n",
"3-Concert / 5-Dance / \n",
"369-Resort / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"16-Performance art / 95-Talent show / \n",
"9-Music video / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"1-Vehicle / 4-Car / 40-Road / 41-Sports car / 70-Driving / 212-Highway / 245-BMW / 344-Coupé / 650-Convertible / 1324-Roadster / 3373-BMW Z4 / \n",
"1-Vehicle / 10-Animal / 18-Outdoor recreation / 42-Fishing / 87-Boat / 91-Fish / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 1263-Glider / \n",
"36-Piano / 53-Keyboard / 265-Electronic keyboard / 2773-iPod nano / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 89-Comics / 185-Dragon Ball / 244-Goku / 418-Gohan / 869-Dragon Ball Z: Budokai Tenkaichi / 876-Freeza / \n",
"3-Concert / \n",
"260-Prayer / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / \n",
"9-Music video / 322-Mixtape / \n",
"7-Musician / 14-Guitar / 24-String instrument / \n",
"0-Games / 2-Video game / 6-Animation / 816-Gundam / 3813-Dynasty Warriors: Gundam / \n",
"84-Snow / 120-Winter / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"0-Games / 2-Video game / 441-The Elder Scrolls / 1410-The Elder Scrolls IV: Oblivion / \n",
"0-Games / 75-Wrestling / \n",
"1-Vehicle / 4-Car / 47-Personal computer / 1996-Intel Core i7 / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / 207-Exhaust system / 2892-GSX-R750 / \n",
"101-Telephone / 178-Samsung Galaxy / 899-Samsung Galaxy S III / \n",
"9-Music video / 541-Black-and-white / \n",
"8-Football / \n",
"3355-Bingo / \n",
"67-Cycling / 69-Bicycle / \n",
"120-Winter / 657-Autumn / \n",
"1-Vehicle / 4-Car / 406-Chevrolet / \n",
"6-Animation / 15-Cartoon / \n",
"322-Mixtape / \n",
"13-Musical ensemble / 820-Bagpipes / 1646-Pipe band / \n",
"8-Football / \n",
"5-Dance / \n",
"5-Dance / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"5-Dance / 67-Cycling / 1997-Unicycle / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 545-Quartet (ensemble) / \n",
"9-Music video / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 398-The Legend of Zelda / 1228-The Legend of Zelda: Ocarina of Time / \n",
"1-Vehicle / 4-Car / 519-Recreational vehicle / \n",
"0-Games / 2-Video game / 2071-Indiana Jones / \n",
"163-Running / \n",
"3-Concert / 7-Musician / 14-Guitar / 30-Drummer / \n",
"31-Disc jockey / \n",
"0-Games / 2-Video game / 51-Strategy video game / 831-Plants vs. Zombies / 1120-Tower defense / \n",
"12-Food / 232-Cake / 1641-Cheesecake / \n",
"3-Concert / \n",
"7-Musician / 13-Musical ensemble / 14-Guitar / 30-Drummer / 419-Earth / 448-Planet / 593-Moon / 654-Sun / 712-Outer space / \n",
"0-Games / 2-Video game / 35-Minecraft / 394-Map / \n",
"0-Games / 1680-Seafight / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 178-Samsung Galaxy / 899-Samsung Galaxy S III / 1207-Samsung Galaxy S5 / 3648-Samsung Galaxy S III Mini / \n",
"5-Dance / 49-School / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"0-Games / 8-Football / 21-Stadium / 73-Ball / \n",
"2-Video game / \n",
"45-Cosmetics / 282-Skin / \n",
"9-Music video / \n",
"2122-Compact Cassette / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / \n",
"5-Dance / 95-Talent show / 115-Ballet / \n",
"25-Toy / 140-LEGO / \n",
"9-Music video / \n",
"3-Concert / 5-Dance / \n",
"0-Games / 2-Video game / 19-PC game / 192-Battlefield / 383-Battlefield 4 / \n",
"6-Animation / 15-Cartoon / 20-Trailer / \n",
"3-Concert / 7-Musician / \n",
"303-Paint / 890-Brush / \n",
"330-Money / \n",
"12-Food / 26-Cooking / 389-Restaurant / \n",
"141-Microsoft Windows / \n",
"3-Concert / 16-Performance art / \n",
"272-iPod / 497-Macintosh / \n",
"10-Animal / 80-Horse / 247-Stallion / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 434-Ninja / 682-PlayStation Vita / 2192-Ninja Gaiden / \n",
"117-Boxing / \n",
"72-iPhone / 272-iPod / 331-iPod touch / \n",
"222-Surfing / \n",
"28-Fashion / 1082-Knot / 1369-Necktie / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / 329-Swimming pool / 639-Fiddle / 670-Banjo / 1231-Mandolin / \n",
"12-Food / \n",
"118-Grand Theft Auto V / 615-Poker / \n",
"76-Ball / 204-Gym / \n",
"12-Food / 585-Lawn / 2437-String trimmer / \n",
"1113-Toilet / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / \n",
"3-Concert / \n",
"7-Musician / 9-Music video / \n",
"1449-Rubber band / \n",
"0-Games / 2-Video game / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 297-Grand Theft Auto IV / 658-Grand Theft Auto: The Lost and Damned / 1923-Niko Bellic / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 41-Sports car / 70-Driving / 94-Dashcam / 212-Highway / 1040-Street racing / 1635-Ferrari 458 / \n",
"1023-Rose / \n",
"10-Animal / 80-Horse / 438-Mare / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 252-Tractor pulling / \n",
"14-Guitar / \n",
"2-Video game / 6-Animation / 229-Halo / 866-Halo 4 / 2889-343 Industries / \n",
"46-Choir / 260-Prayer / 318-Church / 860-Christian Church / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 187-Takeoff / \n",
"61-Art / 97-Drawing / 148-Painting / 1508-Spacecraft / \n",
"215-Eating / 2411-Howcast / \n",
"3-Concert / 5-Dance / \n",
"0-Games / 2-Video game / 55-Video game console / 335-Nintendo 3DS / 1108-Mushroom / 1220-New Super Mario Bros / 3943-New Super Mario Bros. 2 / \n",
"0-Games / 2-Video game / 240-Arcade game / 708-Mermaid / 1876-Beatmania IIDX / 2063-Beatmania / \n",
"22-Nature / \n",
"1-Vehicle / 4-Car / 11-Motorsport / \n",
"1-Vehicle / 124-Tractor / 156-Agriculture / 174-Farm / 249-Heavy equipment / 363-Soil / 459-Plough / 660-Rural area / \n",
"59-Winter sport / 112-Ice skating / 142-Hockey / 234-Ford / \n",
"85-Combat / 117-Boxing / \n",
"95-Talent show / \n",
"0-Games / 35-Minecraft / \n",
"3-Concert / 14-Guitar / 98-Festival / 594-Music festival / \n",
"0-Games / 1-Vehicle / 2-Video game / 413-PlayStation 2 / 1715-Aircraft carrier / \n",
"12-Food / 136-Gardening / 174-Farm / 1108-Mushroom / \n",
"8-Football / \n",
"3-Concert / \n",
"3-Concert / 38-Orchestra / \n",
"5-Dance / 3092-Cinderella / \n",
"5-Dance / 60-Basketball / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"28-Fashion / 696-T-shirt / 1534-H&M / \n",
"6-Animation / 15-Cartoon / \n",
"6-Animation / 15-Cartoon / \n",
"75-Wrestling / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 41-Sports car / 70-Driving / 94-Dashcam / 245-BMW / \n",
"410-Teacher / \n",
"3-Concert / 16-Performance art / \n",
"5-Dance / 28-Fashion / 121-Photography / \n",
"458-Gold / \n",
"3-Concert / \n",
"9-Music video / \n",
"162-Medicine / 1357-Ceramic / \n",
"1-Vehicle / 4-Car / 234-Ford / 1395-Valve / \n",
"2397-Cabbage / \n",
"89-Comics / 294-Batman / 387-Spider-Man / 630-Superman / 2082-Catwoman / \n",
"5-Dance / \n",
"3-Concert / 14-Guitar / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / 265-Electronic keyboard / \n",
"5-Dance / 16-Performance art / 115-Ballet / \n",
"5-Dance / \n",
"61-Art / 97-Drawing / \n",
"9-Music video / \n",
"7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / 46-Choir / \n",
"2-Video game / 43-Call of Duty / 302-Call of Duty 4: Modern Warfare / \n",
"1-Vehicle / 4-Car / \n",
"1-Vehicle / 12-Food / 1793-Taco / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 135-Drifting / \n",
"3-Concert / 68-Lighting / \n",
"12-Food / 22-Nature / 86-Plant / 227-Water / \n",
"0-Games / 395-Camping / \n",
"159-World of Warcraft / 171-Warcraft / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 162-Medicine / 215-Eating / \n",
"0-Games / \n",
"139-Airport / 1061-Marriage proposal / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 2-Video game / 19-PC game / 127-Xbox 360 / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 236-Baking / 274-Meat / 485-Dough / \n",
"6-Animation / \n",
"47-Personal computer / 131-Computer / \n",
"164-Beach / 369-Resort / \n",
"0-Games / 2-Video game / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"0-Games / 8-Football / \n",
"5-Dance / \n",
"6-Animation / 15-Cartoon / 325-Vampire / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 210-Cookware and bakeware / 239-Roasting / 274-Meat / 2650-Frying pan / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 141-Microsoft Windows / 932-Microsoft Lumia / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 378-Drag racing / 1916-Buick / \n",
"0-Games / 2-Video game / 19-PC game / 596-Monster Hunter / \n",
"10-Animal / 48-Pet / 71-Dog / 4389-Basset Hound / \n",
"6-Animation / 15-Cartoon / \n",
"5-Dance / 16-Performance art / \n",
"2-Video game / 34-Action-adventure game / 1396-Lara Croft / \n",
"18-Outdoor recreation / 22-Nature / 275-Diving / 407-Scuba diving / 986-Suit / 4185-Wetsuit / \n",
"28-Fashion / 115-Ballet / 2211-Fur / \n",
"1-Vehicle / 4-Car / 114-Engine / 3187-Spark plug / \n",
"79-American football / \n",
"164-Beach / 2275-Beach volleyball / \n",
"5-Dance / \n",
"762-Angel / \n",
"1551-The Phantom of the Opera / 3904-Miss Saigon / 4443-Jekyll & Hyde / \n",
"117-Boxing / \n",
"1-Vehicle / 4-Car / 154-Wheel / 403-Toyota / 2104-Toyota Camry / \n",
"36-Piano / 53-Keyboard / \n",
"28-Fashion / 45-Cosmetics / 1459-Magazine / \n",
"6-Animation / 20-Trailer / \n",
"125-Television / \n",
"322-Mixtape / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 40-Road / 70-Driving / 201-Grand Theft Auto: San Andreas / 212-Highway / \n",
"3-Concert / 16-Performance art / \n",
"12-Food / 26-Cooking / 52-Dish / 58-Cuisine / 389-Restaurant / \n",
"0-Games / 1802-Jubeat / \n",
"5-Dance / 16-Performance art / 95-Talent show / 115-Ballet / 350-Circus / 788-Juggling / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"3-Concert / 31-Disc jockey / 68-Lighting / 133-Nightclub / \n",
"6-Animation / 61-Art / 97-Drawing / 479-Graffiti / \n",
"616-The Bible / \n",
"1-Vehicle / 4-Car / 245-BMW / 1590-BMW 3 Series (E30) / \n",
"273-News program / \n",
"0-Games / 8-Football / 410-Teacher / \n",
"7-Musician / 14-Guitar / 24-String instrument / \n",
"6-Animation / 123-Naruto / 183-Manga / 285-Sasuke Uchiha / 622-Sakura Haruno / \n",
"0-Games / 2-Video game / 34-Action-adventure game / \n",
"1672-LittleBigPlanet / \n",
"3-Concert / 5-Dance / 31-Disc jockey / 133-Nightclub / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 189-iPad / 272-iPod / 331-iPod touch / \n",
"22-Nature / \n",
"1-Vehicle / 84-Snow / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 392-Radio-controlled helicopter / 393-Unmanned aerial vehicle / 604-Quadcopter / \n",
"0-Games / 1-Vehicle / 2-Video game / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 40-Road / 41-Sports car / 118-Grand Theft Auto V / \n",
"8-Football / \n",
"141-Microsoft Windows / 406-Chevrolet / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / \n",
"835-Backpack / \n",
"0-Games / 4004-Tug of war / \n",
"102-Building / 113-House / 170-Home improvement / 276-Room / 580-Bedroom / 652-Bathroom / 879-Ceiling / 888-Living room / \n",
"0-Games / \n",
"6-Animation / 15-Cartoon / \n",
"125-Television / \n",
"101-Telephone / \n",
"2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 1097-Touchscreen / \n",
"0-Games / 60-Basketball / \n",
"0-Games / 2-Video game / 159-World of Warcraft / 171-Warcraft / 1213-World of Warcraft: Wrath of the Lich King / \n",
"5-Dance / \n",
"44-Drums / 2324-Hang / \n",
"3-Concert / 295-Chocolate / \n",
"273-News program / \n",
"1-Vehicle / 11-Motorsport / 18-Outdoor recreation / 27-Motorcycle / 105-Motorcycling / 197-Trail / \n",
"28-Fashion / 153-Dress / \n",
"12-Food / 26-Cooking / 32-Recipe / 442-Bread / 1943-Bakery / \n",
"319-Robot / 492-Floor / \n",
"28-Fashion / 56-Hair / 106-Hairstyle / 909-Updo / \n",
"242-Paper / 370-Origami / 466-Gift / \n",
"2092-Coal / \n",
"93-Comedy / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 266-Helicopter / 362-PlayStation Portable / 921-Grand Theft Auto: Vice City / 2893-Grand Theft Auto: Vice City Stories / \n",
"0-Games / 2-Video game / 4311-Mega Man Star Force / \n",
"1-Vehicle / 18-Outdoor recreation / 87-Boat / 225-Lake / 481-Kayak / \n",
"311-Street Fighter / 461-Street Fighter IV / \n",
"0-Games / 2-Video game / 257-Counter-Strike / \n",
"5-Dance / 9-Music video / \n",
"7-Musician / 8-Football / 13-Musical ensemble / 21-Stadium / 79-American football / 203-Marching band / 209-University / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 111-PlayStation 3 / 118-Grand Theft Auto V / \n",
"36-Piano / 53-Keyboard / 104-Pianist / \n",
"6-Animation / 15-Cartoon / \n",
"16-Performance art / 115-Ballet / 361-Flamenco / \n",
"9-Music video / \n",
"219-Accordion / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 116-Pokémon / 186-Pokémon / \n",
"2627-Little Red Riding Hood / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 67-Cycling / 69-Bicycle / \n",
"28-Fashion / 45-Cosmetics / \n",
"0-Games / 8-Football / 79-American football / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 711-Gran Turismo / 953-Gran Turismo 5 / \n",
"10-Animal / 15-Cartoon / 116-Pokémon / 596-Monster Hunter / \n",
"31-Disc jockey / 861-Mixing console / \n",
"60-Basketball / 256-Tennis / \n",
"0-Games / 59-Winter sport / 112-Ice skating / 316-Figure skating / 351-Jumping / \n",
"10-Animal / 22-Nature / 147-River / \n",
"3-Concert / \n",
"22-Nature / 267-Mountain / 283-Rock / 800-Mountain pass / \n",
"6-Animation / 25-Toy / 126-The Walt Disney Company / \n",
"97-Drawing / 880-Trousers / 1627-Shorts / \n",
"20-Trailer / \n",
"14-Guitar / 36-Piano / \n",
"371-Firefighter / \n",
"0-Games / 59-Winter sport / 81-Athlete / 112-Ice skating / 142-Hockey / 312-Ice rink / \n",
"5-Dance / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / 1127-Drum stick / \n",
"83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"0-Games / 2-Video game / 116-Pokémon / \n",
"12-Food / 452-Watch / \n",
"9-Music video / \n",
"18-Outdoor recreation / 22-Nature / 287-Human swimming / 329-Swimming pool / \n",
"5-Dance / 38-Orchestra / \n",
"5-Dance / \n",
"8-Football / \n",
"1-Vehicle / 4-Car / 332-Mercedes-Benz / 1119-Sensor / 2603-Mercedes-Benz E-Class / \n",
"0-Games / 116-Pokémon / 186-Pokémon / \n",
"9-Music video / \n",
"5-Dance / 16-Performance art / \n",
"208-Wood / 261-Tool / 400-Woodturning / 1088-Infomercial / \n",
"0-Games / 2-Video game / 43-Call of Duty / 138-Call of Duty: Black Ops II / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / \n",
"5-Dance / \n",
"8-Football / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 88-Machine / 114-Engine / 459-Plough / \n",
"0-Games / 2-Video game / \n",
"47-Personal computer / 228-Laptop / 525-Printer / 696-T-shirt / 1536-Woven fabric / \n",
"0-Games / 59-Winter sport / 81-Athlete / 112-Ice skating / 142-Hockey / 312-Ice rink / \n",
"36-Piano / 446-Sheet music / \n",
"1-Vehicle / 11-Motorsport / 114-Engine / 124-Tractor / 252-Tractor pulling / \n",
"3247-Dome / \n",
"0-Games / 8-Football / 90-Sports game / 163-Running / 180-FIFA 15 / \n",
"31-Disc jockey / \n",
"9-Music video / 14-Guitar / 24-String instrument / \n",
"237-Final Fantasy / 730-Final Fantasy VII / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 856-Airbus A320 family / 4148-Pre-flight safety demonstration / \n",
"8-Football / \n",
"0-Games / 51-Strategy video game / 165-League of Legends / \n",
"6-Animation / 15-Cartoon / 61-Art / 97-Drawing / 616-The Bible / 1075-Whale / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 96-Soldier / 257-Counter-Strike / \n",
"0-Games / 2-Video game / 229-Halo / \n",
"3-Concert / 8-Football / 21-Stadium / 60-Basketball / \n",
"337-Advertising / 798-Shampoo / \n",
"39-Drums / 44-Drums / 2030-Djembe / \n",
"0-Games / 5-Dance / 2123-Just Dance / \n",
"3-Concert / \n",
"28-Fashion / 793-Shirt / 1100-Hat / \n",
"28-Fashion / 56-Hair / 1271-Beauty salon / \n",
"5-Dance / 16-Performance art / \n",
"6-Animation / 15-Cartoon / 2231-Toontown Online / \n",
"0-Games / 8-Football / 90-Sports game / 180-FIFA 15 / \n",
"5-Dance / 892-Aerobics / \n",
"0-Games / 8-Football / 205-Newscaster / 699-Kick / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / \n",
"45-Cosmetics / 238-Nail / 873-Acrylic paint / \n",
"9-Music video / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 162-Medicine / 808-Breakfast / \n",
"0-Games / 21-Stadium / 54-Highlight film / 60-Basketball / 77-Arena / 81-Athlete / 168-Basketball moves / 255-Slam dunk / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 1570-Command & Conquer / 4526-Command & Conquer: Generals / \n",
"56-Hair / 106-Hairstyle / 1320-Bangs / \n",
"2401-Wolfenstein / \n",
"0-Games / 6-Animation / 3367-Bart Simpson / \n",
"18-Outdoor recreation / 22-Nature / \n",
"12-Food / 2992-Energy drink / \n",
"66-Bollywood / \n",
"0-Games / 2-Video game / 55-Video game console / 127-Xbox 360 / 181-Xbox / 199-Wii / 229-Halo / 443-Wii U / 866-Halo 4 / 963-Left 4 Dead 2 / 1140-Assassin's Creed III / \n",
"1-Vehicle / 18-Outdoor recreation / 87-Boat / 949-Rowing / \n",
"25-Toy / 1017-Yo-yo / \n",
"0-Games / 2-Video game / 1144-Castlevania / \n",
"18-Outdoor recreation / 22-Nature / 147-River / 481-Kayak / 1195-Rafting / \n",
"1-Vehicle / 18-Outdoor recreation / 42-Fishing / 87-Boat / 1135-Dolphin / \n",
"3-Concert / 13-Musical ensemble / 16-Performance art / 38-Orchestra / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 17-Racing / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / 352-Need for Speed / \n",
"6-Animation / 89-Comics / 750-Sailor Moon / \n",
"5-Dance / \n",
"1-Vehicle / 4-Car / 94-Dashcam / 1134-Taxicab / \n",
"1-Vehicle / 87-Boat / 147-River / 222-Surfing / 481-Kayak / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 24-String instrument / 36-Piano / 53-Keyboard / 150-Violin / 450-Viola / 1805-Carousel / \n",
"223-Weather / 394-Map / 419-Earth / 448-Planet / 593-Moon / 654-Sun / 1145-Winter storm / 1918-Comet / 3446-Solar flare / \n",
"1-Vehicle / 22-Nature / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 428-Rail transport modelling / \n",
"10-Animal / 71-Dog / \n",
"3-Concert / \n",
"49-School / 102-Building / \n",
"10-Animal / 71-Dog / 80-Horse / 2438-French Bulldog / \n",
"25-Toy / 126-The Walt Disney Company / 417-Play-Doh / \n",
"6-Animation / 432-Kingdom Hearts / 2801-Characters of Kingdom Hearts / 3299-Kingdom Hearts: Chain of Memories / \n",
"9-Music video / \n",
"66-Bollywood / \n",
"1-Vehicle / 259-Fire / 371-Firefighter / \n",
"34-Action-adventure game / 641-Mask / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 297-Grand Theft Auto IV / 658-Grand Theft Auto: The Lost and Damned / \n",
"1-Vehicle / 5-Dance / 11-Motorsport / 124-Tractor / 252-Tractor pulling / \n",
"1-Vehicle / 62-Train / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"9-Music video / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 18-Outdoor recreation / 92-Radio-controlled model / 167-Four-wheel drive / 202-Radio-controlled car / 926-Traxxas / \n",
"1-Vehicle / 4-Car / \n",
"3-Concert / 5-Dance / 10-Animal / 71-Dog / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 139-Airport / 151-Landing / 157-Jet aircraft / 175-Airliner / 187-Takeoff / 188-Airline / 223-Weather / 254-Runway / \n",
"330-Money / 458-Gold / 521-Metal / 731-Coin / 912-Silver / 1426-Treasure / \n",
"20-Trailer / \n",
"78-Wedding / 153-Dress / 190-Bride / \n",
"75-Wrestling / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / \n",
"242-Paper / \n",
"9-Music video / \n",
"3-Concert / 16-Performance art / 68-Lighting / 98-Festival / 594-Music festival / \n",
"8-Football / \n",
"4185-Wetsuit / \n",
"479-Graffiti / \n",
"1-Vehicle / 395-Camping / 914-Tent / \n",
"1-Vehicle / 4-Car / 25-Toy / 74-Truck / 92-Radio-controlled model / 202-Radio-controlled car / \n",
"10-Animal / 48-Pet / 71-Dog / 1377-Golden Retriever / \n",
"3-Concert / 7-Musician / 16-Performance art / 594-Music festival / \n",
"10-Animal / 12-Food / 713-Egg / \n",
"6-Animation / 25-Toy / 140-LEGO / 2312-SpongeBob SquarePants / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / 556-Skateboarding trick / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 520-Bicycle frame / 645-Road bicycle / \n",
"6-Animation / 1523-Alice / 2201-Alice's Adventures in Wonderland / 3342-The Mad Hatter / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 352-Need for Speed / 3252-Burnout Paradise / \n",
"1-Vehicle / 4-Car / 114-Engine / 299-Volkswagen Passenger Cars / \n",
"0-Games / 2-Video game / 33-Weapon / 2379-Tom Clancy's Ghost Recon / 3423-Tom Clancy's Ghost Recon: Future Soldier / \n",
"46-Choir / \n",
"1-Vehicle / 18-Outdoor recreation / 42-Fishing / 87-Boat / 268-Recreational fishing / 709-Bull / \n",
"28-Fashion / 350-Circus / \n",
"447-Sand / 751-Desert / 2410-Camel / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"1-Vehicle / 11-Motorsport / 87-Boat / 2056-Battleship / \n",
"6-Animation / 61-Art / 99-Christmas / 447-Sand / 3282-Sand art and play / 3550-Sand animation / \n",
"110-Album / 4427-Humbucker / \n",
"0-Games / 60-Basketball / 168-Basketball moves / \n",
"118-Grand Theft Auto V / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 96-Soldier / 192-Battlefield / 383-Battlefield 4 / \n",
"125-Television / 205-Newscaster / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"0-Games / 59-Winter sport / 81-Athlete / 112-Ice skating / 142-Hockey / \n",
"22-Nature / 42-Fishing / 164-Beach / 222-Surfing / 233-Ocean / 423-Coast / 672-Bodyboarding / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"3-Concert / 7-Musician / 16-Performance art / 30-Drummer / 39-Drums / 44-Drums / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 256-Tennis / 1626-Forehand / \n",
"0-Games / 21-Stadium / 79-American football / 81-Athlete / \n",
"25-Toy / 140-LEGO / 264-Star Wars / 627-Clock / 749-Figurine / 1245-Alarm device / 3429-Yoda / 3551-Alarm clock / \n",
"5-Dance / 115-Ballet / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / \n",
"3-Concert / 16-Performance art / \n",
"1-Vehicle / 4-Car / 270-Sedan / \n",
"37-Gadget / 47-Personal computer / 145-Tablet computer / \n",
"42-Fishing / 510-Fishing lure / 902-Fishing bait / \n",
"1-Vehicle / 4-Car / \n",
"14-Guitar / 1972-Toy balloon / \n",
"6-Animation / 356-Star / 411-MapleStory / \n",
"230-Brass instrument / 427-Trumpet / 4673-Flugelhorn / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 172-Motocross / 197-Trail / \n",
"42-Fishing / 548-Parachuting / 573-Paragliding / 765-Parachute / \n",
"205-Newscaster / 371-Firefighter / \n",
"5-Dance / \n",
"1788-Fencing / \n",
"6-Animation / 995-Attack on Titan / \n",
"93-Comedy / \n",
"3-Concert / \n",
"9-Music video / \n",
"1089-Wire / 1616-Electrical wiring / 1864-Switch / \n",
"9-Music video / 99-Christmas / \n",
"3-Concert / 357-DVD / \n",
"694-Tai chi / \n",
"10-Animal / 18-Outdoor recreation / 280-Wildlife / 2396-Coyote / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 216-Call of Duty: Modern Warfare 3 / \n",
"61-Art / 97-Drawing / 148-Painting / 242-Paper / 303-Paint / \n",
"86-Plant / 136-Gardening / \n",
"0-Games / 2-Video game / 1170-Super Mario World / \n",
"7-Musician / 13-Musical ensemble / 22-Nature / 46-Choir / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 158-Tire / 3548-Yamaha YZ125 / \n",
"113-House / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 114-Engine / 2286-Grand Tourer Injection / 2826-Suzuki Swift / \n",
"3-Concert / \n",
"20-Trailer / 60-Basketball / \n",
"75-Wrestling / 85-Combat / \n",
"6-Animation / 15-Cartoon / \n",
"93-Comedy / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 96-Soldier / 114-Engine / 257-Counter-Strike / 266-Helicopter / \n",
"3-Concert / \n",
"93-Comedy / \n",
"38-Orchestra / 356-Star / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / 361-Flamenco / 3369-Diego de la Vega / \n",
"2019-Dental braces / \n",
"395-Camping / \n",
"0-Games / 49-School / 54-Highlight film / 60-Basketball / 168-Basketball moves / \n",
"6-Animation / 15-Cartoon / 185-Dragon Ball / 244-Goku / 418-Gohan / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 378-Drag racing / \n",
"18-Outdoor recreation / 42-Fishing / 84-Snow / 120-Winter / 262-Ice / 287-Human swimming / \n",
"66-Bollywood / 125-Television / \n",
"20-Trailer / 286-Bus / \n",
"6-Animation / 102-Building / 113-House / 416-Apartment / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 175-Airliner / 188-Airline / \n",
"6-Animation / 15-Cartoon / 547-Bleach / \n",
"9-Music video / \n",
"6-Animation / 15-Cartoon / 67-Cycling / 69-Bicycle / 126-The Walt Disney Company / 4323-Cricket / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 328-Simulation video game / 1407-Animal Crossing / 1819-Animal Crossing: New Leaf / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / \n",
"1-Vehicle / 4-Car / 1950-The CW / \n",
"0-Games / 47-Personal computer / 1169-Hot Wheels / \n",
"1-Vehicle / 62-Train / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 20-Trailer / 2398-Personal armor / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 42-Fishing / 91-Fish / \n",
"22-Nature / 86-Plant / 1023-Rose / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"298-Television advertisement / 2032-Pepsi / \n",
"6-Animation / 15-Cartoon / 126-The Walt Disney Company / \n",
"1-Vehicle / 4-Car / 49-School / 64-Transport / 74-Truck / 286-Bus / 2199-School bus / \n",
"898-Pilates / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / 412-Monster / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 790-Nissan GT-R / 1699-Cart / \n",
"0-Games / 2-Video game / 4392-The Evil Within / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 135-Drifting / 245-BMW / 344-Coupé / \n",
"599-Dofus / \n",
"8-Football / \n",
"20-Trailer / \n",
"12-Food / 26-Cooking / 32-Recipe / \n",
"93-Comedy / \n",
"1-Vehicle / 4-Car / 25-Toy / 344-Coupé / \n",
"6-Animation / 15-Cartoon / \n",
"10-Animal / 280-Wildlife / 425-City / \n",
"2-Video game / 387-Spider-Man / \n",
"0-Games / 458-Gold / 1787-Happy Farm / \n",
"0-Games / 2-Video game / 43-Call of Duty / 47-Personal computer / 194-Call of Duty: Modern Warfare 2 / 302-Call of Duty 4: Modern Warfare / 431-Call of Duty: Advanced Warfare / 464-First-person Shooter / 602-Central processing unit / \n",
"597-Soup / \n",
"47-Personal computer / 228-Laptop / 512-Printing / 525-Printer / \n",
"2009-Electromagnetic coil / 3806-Tesla coil / \n",
"0-Games / 89-Comics / 781-Pachinko / \n",
"2484-Electronic circuit / 3285-Boombox / \n",
"5-Dance / 16-Performance art / 115-Ballet / \n",
"130-Gymnastics / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"287-Human swimming / 329-Swimming pool / \n",
"2-Video game / 237-Final Fantasy / \n",
"163-Running / \n",
"7-Musician / 36-Piano / 53-Keyboard / 104-Pianist / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"13-Musical ensemble / 49-School / 98-Festival / 203-Marching band / 250-Parade / \n",
"45-Cosmetics / 436-Concealer / \n",
"93-Comedy / 101-Telephone / \n",
"5-Dance / 8-Football / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 47-Personal computer / 145-Tablet computer / \n",
"164-Beach / 447-Sand / 573-Paragliding / 975-Dune / \n",
"99-Christmas / 2649-Nativity scene / \n",
"6-Animation / 61-Art / 89-Comics / 183-Manga / 547-Bleach / \n",
"42-Fishing / \n",
"14-Guitar / 219-Accordion / 655-Diatonic button accordion / \n",
"191-Cheerleading / \n",
"5-Dance / 2198-Irish dance / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"0-Games / 6-Animation / 1563-Stick figure / \n",
"616-The Bible / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"25-Toy / 524-Littlest Pet Shop / 749-Figurine / \n",
"5-Dance / \n",
"25-Toy / 89-Comics / 435-Action figure / 749-Figurine / 2121-Marvel Legends / 2735-Raccoon / \n",
"0-Games / 734-DayZ / 736-ARMA / 1216-DayZ / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 235-Lipstick / 291-Rouge / 436-Concealer / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 245-BMW / 270-Sedan / 874-BMW M3 / 1709-BMW 3 Series (E46) / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 70-Driving / 94-Dashcam / \n",
"0-Games / 2-Video game / 20-Trailer / 1991-Rappelz / \n",
"144-Amusement park / 334-Amusement ride / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"97-Drawing / \n",
"3-Concert / \n",
"9-Music video / \n",
"3-Concert / \n",
"98-Festival / 250-Parade / 253-Carnival / \n",
"260-Prayer / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"0-Games / 5-Dance / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"1054-Vacuum cleaner / \n",
"20-Trailer / 264-Star Wars / \n",
"31-Disc jockey / 861-Mixing console / 3458-DMC World DJ Championships / \n",
"42-Fishing / \n",
"28-Fashion / 153-Dress / \n",
"925-Brain / \n",
"28-Fashion / \n",
"5-Dance / 250-Parade / \n",
"14-Guitar / \n",
"9-Music video / \n",
"7-Musician / 14-Guitar / 24-String instrument / \n",
"0-Games / 85-Combat / 116-Pokémon / 186-Pokémon / \n",
"250-Parade / 253-Carnival / \n",
"3-Concert / 7-Musician / 30-Drummer / 38-Orchestra / 39-Drums / 44-Drums / \n",
"2-Video game / 34-Action-adventure game / 55-Video game console / 111-PlayStation 3 / 176-PlayStation 4 / 317-PlayStation / 950-Blu-ray disc / 1985-The Last of Us / \n",
"1-Vehicle / 4-Car / 28-Fashion / 785-Jeans / 1534-H&M / \n",
"0-Games / 19-PC game / 35-Minecraft / \n",
"1-Vehicle / 156-Agriculture / 174-Farm / 715-Farming Simulator / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"1-Vehicle / 4-Car / 945-MINI Cooper / \n",
"3-Concert / \n",
"162-Medicine / 3971-Dentures / \n",
"28-Fashion / \n",
"20-Trailer / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 377-BMX bike / \n",
"14-Guitar / 44-Drums / 1825-Cajón / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"61-Art / 479-Graffiti / \n",
"6-Animation / 15-Cartoon / \n",
"5-Dance / \n",
"9-Music video / \n",
"5-Dance / 769-Red Bull / \n",
"5-Dance / \n",
"10-Animal / 48-Pet / 166-Cat / \n",
"162-Medicine / \n",
"9-Music video / \n",
"9-Music video / \n",
"5-Dance / \n",
"3-Concert / 7-Musician / 14-Guitar / 30-Drummer / \n",
"1-Vehicle / 18-Outdoor recreation / 87-Boat / \n",
"12-Food / 25-Toy / 215-Eating / \n",
"3-Concert / \n",
"0-Games / 411-MapleStory / \n",
"22-Nature / \n",
"0-Games / 1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 3504-Honda NSX / 4302-GTR 2 – FIA GT Racing Game / \n",
"162-Medicine / 196-Eye / 625-Factory / \n",
"28-Fashion / 153-Dress / \n",
"463-Cricket / 1495-Twenty20 / \n",
"47-Personal computer / 131-Computer / 145-Tablet computer / 228-Laptop / 2615-HP Pavilion / \n",
"25-Toy / 354-Egg / 439-Barbie / 611-Kinder Surprise / 1047-Easter egg / \n",
"6-Animation / 126-The Walt Disney Company / \n",
"20-Trailer / 222-Surfing / \n",
"102-Building / 318-Church / 885-Bell / \n",
"0-Games / 49-School / 54-Highlight film / 60-Basketball / 81-Athlete / 168-Basketball moves / 255-Slam dunk / 306-High school / 1101-Point guard / \n",
"10-Animal / 22-Nature / 91-Fish / 258-Aquarium / 865-Turtle / \n",
"102-Building / 113-House / 170-Home improvement / 276-Room / 888-Living room / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"6-Animation / 102-Building / 113-House / \n",
"13-Musical ensemble / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 40-Road / \n",
"5-Dance / 16-Performance art / 95-Talent show / \n",
"35-Minecraft / 1149-Castle / \n",
"0-Games / 1837-Realm of the Mad God / \n",
"3-Concert / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"6-Animation / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 76-Ball / \n",
"260-Prayer / \n",
"5-Dance / 16-Performance art / 78-Wedding / 190-Bride / \n",
"15-Cartoon / 89-Comics / \n",
"5-Dance / 16-Performance art / 95-Talent show / \n",
"5-Dance / 410-Teacher / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"125-Television / \n",
"1-Vehicle / 27-Motorcycle / 516-Headphones / 522-Helmet / 832-Headset / \n",
"163-Running / \n",
"1-Vehicle / 4-Car / 9-Music video / 60-Basketball / 255-Slam dunk / \n",
"12-Food / 26-Cooking / 32-Recipe / 88-Machine / 539-Home appliance / 1489-Coffeemaker / 1729-Latte / 1823-Espresso machine / 2057-Cappuccino / \n",
"46-Choir / \n",
"12-Food / \n",
"1-Vehicle / 40-Road / 74-Truck / 2226-Bus driver / \n",
"8-Football / \n",
"35-Minecraft / 61-Art / 2945-Fan art / \n",
"46-Choir / 318-Church / 870-Chapel / \n",
"0-Games / 2-Video game / 51-Strategy video game / 1989-Oculus Rift / \n",
"121-Photography / 155-Camera / 566-Digital camera / 1855-Video camera / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"10-Animal / 80-Horse / \n",
"96-Soldier / 794-Bridge / \n",
"5-Dance / 16-Performance art / \n",
"0-Games / 10-Animal / 166-Cat / 1193-Orange / \n",
"5-Dance / 16-Performance art / 39-Drums / \n",
"0-Games / 1496-LittleBigPlanet 2 / 1672-LittleBigPlanet / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 179-Kitchen / 301-Meal / \n",
"1608-Injury / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 231-Supercar / 1324-Roadster / 1486-Bugatti Veyron / 1875-Bugatti / \n",
"2914-Wreath / \n",
"5-Dance / 93-Comedy / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 282-Skin / 291-Rouge / 436-Concealer / \n",
"42-Fishing / 84-Snow / 120-Winter / 262-Ice / \n",
"47-Personal computer / 145-Tablet computer / 847-Angry Birds / \n",
"6-Animation / 708-Mermaid / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 114-Engine / 332-Mercedes-Benz / \n",
"1623-Shaolin Kung Fu / \n",
"10-Animal / 22-Nature / 91-Fish / 275-Diving / 288-Underwater / 381-Underwater diving / 407-Scuba diving / 716-Coral / \n",
"1-Vehicle / 74-Truck / 92-Radio-controlled model / \n",
"23-Mobile phone / 29-Smartphone / 125-Television / 487-Samsung Electronics / 1595-Television set / \n",
"162-Medicine / 214-Muscle / \n",
"0-Games / 2-Video game / 505-Mega Man / 581-PlayStation / \n",
"43-Call of Duty / 216-Call of Duty: Modern Warfare 3 / \n",
"15-Cartoon / 61-Art / 97-Drawing / 424-Chicken / \n",
"59-Winter sport / 84-Snow / 120-Winter / 177-Skiing / 281-Snowboarding / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 434-Ninja / 2192-Ninja Gaiden / \n",
"10-Animal / 14-Guitar / 48-Pet / 132-Bird / 1147-Hamster / \n",
"0-Games / 54-Highlight film / 60-Basketball / 77-Arena / \n",
"75-Wrestling / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 431-Call of Duty: Advanced Warfare / \n",
"8-Football / \n",
"8-Football / \n",
"1-Vehicle / 4-Car / 124-Tractor / 542-Potato / 2079-Two-wheel tractor / \n",
"5-Dance / \n",
"1-Vehicle / 62-Train / 119-Locomotive / 143-Railroad car / 161-Train station / \n",
"61-Art / 97-Drawing / 3544-Rubber stamp / \n",
"28-Fashion / \n",
"153-Dress / \n",
"3-Concert / 7-Musician / \n",
"12-Food / 88-Machine / 625-Factory / \n",
"61-Art / 82-Airplane / 242-Paper / 313-Book / 370-Origami / 1933-Paper plane / \n",
"9-Music video / \n",
"12-Food / 26-Cooking / 179-Kitchen / \n",
"512-Printing / 525-Printer / 1063-Ink / \n",
"5-Dance / 95-Talent show / \n",
"0-Games / 8-Football / \n",
"3-Concert / 16-Performance art / 31-Disc jockey / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 73-Ball / 79-American football / 81-Athlete / \n",
"93-Comedy / \n",
"1106-Character / \n",
"20-Trailer / 89-Comics / 2069-Flash / \n",
"9-Music video / \n",
"28-Fashion / 1237-Baby carriage / \n",
"22-Nature / 86-Plant / 136-Gardening / 807-Seed / 1460-Herb / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 513-Computer keyboard / \n",
"56-Hair / 583-Wig / 733-Washing / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"25-Toy / 435-Action figure / 735-Kamen Rider Series / 2525-Gashapon / \n",
"99-Christmas / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / \n",
"121-Photography / 541-Black-and-white / \n",
"37-Gadget / 47-Personal computer / 131-Computer / 145-Tablet computer / 189-iPad / \n",
"7-Musician / 9-Music video / 14-Guitar / \n",
"6-Animation / 9-Music video / 15-Cartoon / 97-Drawing / \n",
"12-Food / 944-Hamburger / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 35-Minecraft / \n",
"79-American football / 465-Microphone / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 21-Stadium / 27-Motorcycle / 105-Motorcycling / \n",
"2649-Nativity scene / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 172-Motocross / 447-Sand / \n",
"13-Musical ensemble / \n",
"75-Wrestling / 85-Combat / \n",
"84-Snow / 85-Combat / 120-Winter / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"42-Fishing / 287-Human swimming / 329-Swimming pool / 892-Aerobics / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / 224-Rapid transit / \n",
"1-Vehicle / 50-Aircraft / 129-Aviation / 736-ARMA / \n",
"9-Music video / \n",
"5-Dance / 387-Spider-Man / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 1985-The Last of Us / \n",
"18-Outdoor recreation / 59-Winter sport / 177-Skiing / 789-Alpine skiing / \n",
"9-Music video / 995-Attack on Titan / \n",
"3-Concert / 1038-Hijab / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 105-Motorcycling / 378-Drag racing / \n",
"2-Video game / 127-Xbox 360 / 704-Call of Duty: World at War / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / \n",
"0-Games / 2-Video game / 19-PC game / 1566-Doom / \n",
"372-Playing card / 746-Card manipulation / \n",
"10-Animal / 42-Fishing / 91-Fish / 275-Diving / 381-Underwater diving / \n",
"4004-Tug of war / \n",
"12-Food / 2040-Korean food / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 199-Wii / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"7-Musician / 24-String instrument / 150-Violin / 842-Harp / \n",
"14-Guitar / 24-String instrument / \n",
"0-Games / 51-Strategy video game / 174-Farm / 373-Clash of Clans / \n",
"10-Animal / 12-Food / 22-Nature / 280-Wildlife / \n",
"42-Fishing / \n",
"372-Playing card / 746-Card manipulation / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"3-Concert / 31-Disc jockey / \n",
"66-Bollywood / \n",
"1-Vehicle / 4-Car / 18-Outdoor recreation / 22-Nature / 167-Four-wheel drive / 197-Trail / 198-Off-road vehicle / 360-Jeep / 1205-Jeep Wrangler / \n",
"110-Album / 567-Compact disc / \n",
"5-Dance / 16-Performance art / 46-Choir / \n",
"10-Animal / 540-Insect / 775-Bee / 784-Honey / 1121-Beehive / \n",
"47-Personal computer / 141-Microsoft Windows / \n",
"3-Concert / \n",
"86-Plant / 136-Gardening / 156-Agriculture / 174-Farm / 906-Sowing / 1678-Foam / \n",
"6-Animation / 15-Cartoon / \n",
"341-Sketch comedy / \n",
"0-Games / 2-Video game / 127-Xbox 360 / 511-Mortal Kombat / 1756-Predator / 2011-Alien / 2837-Killer Instinct / 3255-Aliens vs. Predator / 4046-Killer Instinct / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 257-Counter-Strike / 702-Counter-Strike: Source / \n",
"0-Games / 2-Video game / 1630-Warframe / \n",
"3-Concert / 31-Disc jockey / 133-Nightclub / \n",
"1802-Jubeat / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"5-Dance / 115-Ballet / \n",
"2-Video game / 20-Trailer / 111-PlayStation 3 / 1928-BioShock / 2645-BioShock Infinite / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 27-Motorcycle / 40-Road / 105-Motorcycling / 864-Speedometer / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 1400-rFactor / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 147-River / 197-Trail / 1694-Harness racing / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 3679-Super GT / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"0-Games / 2-Video game / 51-Strategy video game / 165-League of Legends / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 76-Ball / 79-American football / \n",
"814-Pond / \n",
"0-Games / 2-Video game / 2267-Geometry Dash / \n",
"5-Dance / 8-Football / \n",
"9-Music video / \n",
"12-Food / 26-Cooking / 58-Cuisine / 122-Weight training / 179-Kitchen / \n",
"1-Vehicle / 10-Animal / 18-Outdoor recreation / 22-Nature / 42-Fishing / 87-Boat / 91-Fish / 268-Recreational fishing / 510-Fishing lure / \n",
"0-Games / 35-Minecraft / \n",
"3-Concert / 13-Musical ensemble / 16-Performance art / 68-Lighting / \n",
"31-Disc jockey / \n",
"12-Food / 26-Cooking / 32-Recipe / 210-Cookware and bakeware / 391-Chicken meat / 542-Potato / 943-Stove / \n",
"25-Toy / 113-House / 200-Doll / 246-Furniture / 1842-Dollhouse / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / \n",
"10-Animal / 136-Gardening / 1384-Greenhouse / \n",
"25-Toy / 200-Doll / 232-Cake / 323-Toddler / 1192-Hello Kitty / \n",
"5-Dance / 16-Performance art / \n",
"3-Concert / \n",
"10-Animal / 18-Outdoor recreation / 42-Fishing / 91-Fish / 275-Diving / 288-Underwater / 381-Underwater diving / 407-Scuba diving / 3910-Mackerel / \n",
"8-Football / 298-Television advertisement / \n",
"3-Concert / 5-Dance / 8-Football / 769-Red Bull / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 179-Kitchen / 557-Salad / 771-Noodle / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 207-Exhaust system / \n",
"1-Vehicle / \n",
"1-Vehicle / 27-Motorcycle / \n",
"1-Vehicle / 62-Train / 103-Rail transport / 152-Track / 161-Train station / \n",
"12-Food / 26-Cooking / 389-Restaurant / \n",
"0-Games / 35-Minecraft / \n",
"2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 155-Camera / \n",
"6-Animation / 15-Cartoon / 1744-Black Butler / \n",
"5-Dance / 16-Performance art / 95-Talent show / \n",
"0-Games / 2-Video game / 6-Animation / 37-Gadget / 55-Video game console / 292-Handheld game console / 335-Nintendo 3DS / 995-Attack on Titan / \n",
"0-Games / 2-Video game / 19-PC game / 229-Halo / 470-Halo 3 / 644-Halo: Reach / 1161-Halo 2 / 2002-Halo 3: ODST / \n",
"1-Vehicle / 12-Food / 215-Eating / 2190-Bowl / \n",
"43-Call of Duty / 138-Call of Duty: Black Ops II / \n",
"0-Games / 159-World of Warcraft / 171-Warcraft / \n",
"3-Concert / 16-Performance art / \n",
"75-Wrestling / 777-WWE 2K / 977-WWE '13 / 1582-WWE 2K14 / \n",
"20-Trailer / \n",
"9-Music video / \n",
"6-Animation / \n",
"117-Boxing / \n",
"385-Face / \n",
"28-Fashion / 49-School / \n",
"61-Art / 148-Painting / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 105-Motorcycling / 172-Motocross / 198-Off-road vehicle / 669-Extreme sport / 728-Dirt track racing / \n",
"3-Concert / \n",
"0-Games / 10-Animal / \n",
"61-Art / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 129-Aviation / 139-Airport / 157-Jet aircraft / 175-Airliner / 188-Airline / 254-Runway / 856-Airbus A320 family / \n",
"2-Video game / 9-Music video / 43-Call of Duty / 134-Call of Duty: Black Ops / 229-Halo / 1580-Crysis 2 / \n",
"0-Games / 326-Metin2 / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 24-String instrument / 38-Orchestra / 150-Violin / \n",
"0-Games / 59-Winter sport / 112-Ice skating / 142-Hockey / \n",
"8-Football / \n",
"18-Outdoor recreation / 22-Nature / 347-Climbing / 3318-Abseiling / \n",
"1-Vehicle / 4-Car / 41-Sports car / 70-Driving / 299-Volkswagen Passenger Cars / 668-Volkswagen Golf / \n",
"5-Dance / \n",
"6-Animation / \n",
"293-Slide show / \n",
"0-Games / 269-Comic book / 489-Board game / 922-Tabletop game / 2833-Go / \n",
"3-Concert / 7-Musician / 1684-Hillsong Church / \n",
"0-Games / 362-PlayStation Portable / \n",
"2-Video game / 37-Gadget / 55-Video game console / 292-Handheld game console / 335-Nintendo 3DS / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 651-Stock car racing / \n",
"9-Music video / \n",
"3-Concert / 7-Musician / 16-Performance art / 30-Drummer / \n",
"8-Football / 595-Champion / \n",
"66-Bollywood / \n",
"6-Animation / 10-Animal / 15-Cartoon / 71-Dog / \n",
"117-Boxing / \n",
"9-Music video / 98-Festival / \n",
"25-Toy / 200-Doll / \n",
"38-Orchestra / 446-Sheet music / \n",
"3-Concert / \n",
"3-Concert / 7-Musician / 4209-Scarlet / \n",
"325-Vampire / \n",
"10-Animal / 71-Dog / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 2-Video game / 19-PC game / 192-Battlefield / 374-Battlefield 3 / \n",
"61-Art / 97-Drawing / 148-Painting / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / 216-Call of Duty: Modern Warfare 3 / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"0-Games / 19-PC game / 211-RuneScape / 510-Fishing lure / \n",
"0-Games / 8-Football / 49-School / \n",
"49-School / \n",
"211-RuneScape / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 824-Terrier / \n",
"0-Games / 51-Strategy video game / 373-Clash of Clans / \n",
"125-Television / \n",
"162-Medicine / \n",
"88-Machine / 208-Wood / 261-Tool / 882-Pen / 2428-Barn / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 114-Engine / 336-Scooter / 2308-Belt drive / \n",
"9-Music video / \n",
"1-Vehicle / 88-Machine / 114-Engine / 533-Oil / \n",
"0-Games / 49-School / 54-Highlight film / 60-Basketball / 168-Basketball moves / 1101-Point guard / \n",
"9-Music video / \n",
"5-Dance / \n",
"1-Vehicle / 12-Food / 124-Tractor / 156-Agriculture / 174-Farm / 542-Potato / \n",
"0-Games / 1-Vehicle / 4-Car / 41-Sports car / 2400-Asphalt / \n",
"45-Cosmetics / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / \n",
"2-Video game / 6-Animation / 20-Trailer / 1928-BioShock / 2645-BioShock Infinite / \n",
"5-Dance / 115-Ballet / 1253-Ballet Dancer / \n",
"313-Book / \n",
"5-Dance / 31-Disc jockey / \n",
"93-Comedy / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 184-Vegetable / 475-Cheese / 557-Salad / \n",
"18-Outdoor recreation / 22-Nature / 86-Plant / 136-Gardening / 137-Tree / 213-Forest / \n",
"233-Ocean / \n",
"1-Vehicle / 11-Motorsport / 124-Tractor / 252-Tractor pulling / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / 44-Drums / \n",
"9-Music video / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 30-Drummer / 44-Drums / \n",
"546-Stitch / 4246-Crochet hook / \n",
"5-Dance / \n",
"10-Animal / 48-Pet / 537-Zoo / 970-Monkey / \n",
"9-Music video / 78-Wedding / 121-Photography / 190-Bride / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 89-Comics / \n",
"8-Football / \n",
"9-Music video / \n",
"36-Piano / 53-Keyboard / 446-Sheet music / \n",
"0-Games / 2-Video game / \n",
"7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / 870-Chapel / \n",
"354-Egg / 1924-Symbol / 2227-Multimedios Televisión / \n",
"130-Gymnastics / 1030-Trampoline / 1335-Trampolining / \n",
"0-Games / 2277-The Lord of the Rings Online / \n",
"43-Call of Duty / 229-Halo / 470-Halo 3 / \n",
"0-Games / 6-Animation / 15-Cartoon / 837-Roblox / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 70-Driving / 94-Dashcam / 449-Honda / 552-Honda Civic / 1493-Supercharger / 3697-Honda K engine / \n",
"2-Video game / 20-Trailer / 34-Action-adventure game / 35-Minecraft / 380-Assassin's Creed / \n",
"10-Animal / 80-Horse / \n",
"6-Animation / \n",
"0-Games / 6-Animation / 8-Football / 90-Sports game / \n",
"3-Concert / 31-Disc jockey / 133-Nightclub / 988-Ibiza / \n",
"12-Food / 26-Cooking / 1227-Sushi / \n",
"1-Vehicle / 12-Food / 156-Agriculture / 617-Maize / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / 3014-Gibson SG / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / 371-Firefighter / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 118-Grand Theft Auto V / 277-Sport utility vehicle / \n",
"3-Concert / 5-Dance / 49-School / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 509-Classic car / 2558-Plymouth / \n",
"36-Piano / \n",
"4007-The House of the Dead / \n",
"0-Games / 257-Counter-Strike / \n",
"3-Concert / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 70-Driving / 207-Exhaust system / 234-Ford / 344-Coupé / 509-Classic car / 1148-Hot rod / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 691-Cargo / \n",
"102-Building / 113-House / 276-Room / 462-Table / 888-Living room / \n",
"66-Bollywood / 905-Flash Video / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 352-Need for Speed / 1770-Need for Speed: Hot Pursuit / 2545-Need for Speed III: Hot Pursuit / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 169-Rallying / \n",
"9-Music video / \n",
"0-Games / 59-Winter sport / 112-Ice skating / 142-Hockey / \n",
"13-Musical ensemble / 44-Drums / 2444-Crêpe / 3221-Pizzica / \n",
"141-Microsoft Windows / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 73-Ball / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 158-Tire / 1148-Hot rod / 2917-Rat rod / \n",
"3-Concert / \n",
"10-Animal / 80-Horse / 247-Stallion / 289-Dressage / 438-Mare / \n",
"3-Concert / 1386-ABS-CBN / \n",
"18-Outdoor recreation / 42-Fishing / 267-Mountain / 800-Mountain pass / \n",
"3-Concert / \n",
"10-Animal / 20-Trailer / 71-Dog / \n",
"3-Concert / 7-Musician / \n",
"6-Animation / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 40-Road / 105-Motorcycling / \n",
"0-Games / 2-Video game / 1288-Marvel vs. Capcom / 1499-Marvel vs. Capcom 3: Fate of Two Worlds / 3081-Marvel vs. Capcom 2: New Age of Heroes / \n",
"28-Fashion / 206-Shoe / 368-Sneakers / 2257-Athletic shoe / \n",
"7-Musician / 14-Guitar / \n",
"2-Video game / 432-Kingdom Hearts / 1908-Kingdom Hearts Birth by Sleep / \n",
"28-Fashion / \n",
"0-Games / 2-Video game / 505-Mega Man / \n",
"6-Animation / 126-The Walt Disney Company / 996-Princess / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"0-Games / 2-Video game / 20-Trailer / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 687-Taiko no Tatsujin / \n",
"93-Comedy / \n",
"13-Musical ensemble / \n",
"28-Fashion / \n",
"8-Football / \n",
"1-Vehicle / 4-Car / \n",
"15-Cartoon / 185-Dragon Ball / 244-Goku / 418-Gohan / 1304-Cell / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / 1103-Gibson Les Paul / \n",
"2-Video game / 23-Mobile phone / 29-Smartphone / 72-iPhone / 189-iPad / 272-iPod / \n",
"3-Concert / 68-Lighting / 263-Light / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"75-Wrestling / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 64-Transport / 74-Truck / 378-Drag racing / 482-Asphalt / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 135-Drifting / 378-Drag racing / \n",
"6-Animation / 15-Cartoon / \n",
"31-Disc jockey / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / \n",
"75-Wrestling / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 84-Snow / 94-Dashcam / 120-Winter / 158-Tire / \n",
"4511-Watchmen / \n",
"56-Hair / \n",
"3-Concert / 16-Performance art / 68-Lighting / \n",
"66-Bollywood / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"5-Dance / \n",
"15-Cartoon / 921-Grand Theft Auto: Vice City / \n",
"5-Dance / 16-Performance art / 115-Ballet / 577-Penguin / \n",
"6-Animation / 185-Dragon Ball / 244-Goku / \n",
"14-Guitar / 606-String / 1971-Epiphone / \n",
"20-Trailer / \n",
"3-Concert / 5-Dance / 16-Performance art / 31-Disc jockey / 133-Nightclub / \n",
"1697-Astrological sign / \n",
"5-Dance / 191-Cheerleading / 776-Kindergarten / \n",
"0-Games / 21-Stadium / 49-School / 81-Athlete / 562-Pitcher / 946-Baseball park / \n",
"6-Animation / 15-Cartoon / \n",
"117-Boxing / 595-Champion / \n",
"20-Trailer / 541-Black-and-white / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / \n",
"208-Wood / 400-Woodturning / \n",
"0-Games / \n",
"110-Album / \n",
"0-Games / 2-Video game / 535-Slot machine / \n",
"3-Concert / \n",
"20-Trailer / 1014-Movieclips / \n",
"5-Dance / 68-Lighting / 102-Building / 227-Water / 263-Light / 804-Shopping mall / 1222-Fountain / \n",
"2297-Elf / \n",
"6-Animation / 99-Christmas / 357-DVD / \n",
"0-Games / 2-Video game / 257-Counter-Strike / \n",
"9-Music video / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"325-Vampire / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 81-Athlete / \n",
"38-Orchestra / \n",
"7-Musician / 9-Music video / 14-Guitar / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / 265-Electronic keyboard / \n",
"1-Vehicle / 4-Car / \n",
"1-Vehicle / 78-Wedding / \n",
"8-Football / \n",
"61-Art / 148-Painting / 936-Watercolor paint / \n",
"7-Musician / 13-Musical ensemble / 14-Guitar / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / 4165-Washburn Guitars / \n",
"693-Sword / \n",
"12-Food / 170-Home improvement / 652-Bathroom / 1592-Plumbing / 2153-Sink / \n",
"1262-NBA 2K14 / \n",
"7-Musician / 150-Violin / \n",
"121-Photography / 512-Printing / 633-Photograph / \n",
"148-Painting / \n",
"8-Football / 54-Highlight film / \n",
"10-Animal / 166-Cat / 196-Eye / 283-Rock / \n",
"23-Mobile phone / 29-Smartphone / 513-Computer keyboard / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 336-Scooter / 1036-Moped / \n",
"0-Games / 2-Video game / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / \n",
"3-Concert / 5-Dance / 31-Disc jockey / 133-Nightclub / \n",
"6-Animation / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 608-Wheelie / 927-Supermoto / \n",
"31-Disc jockey / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 603-StarCraft II: Wings of Liberty / 732-StarCraft / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / 131-Computer / 265-Electronic keyboard / \n",
"14-Guitar / 192-Battlefield / 1370-Battlefield 2 / \n",
"3-Concert / 68-Lighting / 263-Light / \n",
"8-Football / \n",
"9-Music video / 66-Bollywood / \n",
"1-Vehicle / 4-Car / \n",
"28-Fashion / 153-Dress / \n",
"8-Football / \n",
"89-Comics / \n",
"3-Concert / 7-Musician / 30-Drummer / 133-Nightclub / 1280-Club / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 553-Sony Xperia / \n",
"3804-Music of Eritrea / \n",
"5-Dance / 16-Performance art / \n",
"0-Games / 8-Football / \n",
"7-Musician / 14-Guitar / 24-String instrument / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / \n",
"130-Gymnastics / 154-Wheel / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 40-Road / \n",
"23-Mobile phone / 29-Smartphone / 465-Microphone / \n",
"0-Games / 2-Video game / 55-Video game console / \n",
"9-Music video / \n",
"10-Animal / 865-Turtle / \n",
"36-Piano / \n",
"9-Music video / \n",
"22-Nature / \n",
"1-Vehicle / 27-Motorcycle / 99-Christmas / \n",
"66-Bollywood / \n",
"3497-Hair dryer / \n",
"9-Music video / \n",
"0-Games / 8-Football / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / 670-Banjo / \n",
"93-Comedy / 355-Comedian / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / 199-Wii / \n",
"6-Animation / 25-Toy / 200-Doll / \n",
"1-Vehicle / 62-Train / 103-Rail transport / 152-Track / 428-Rail transport modelling / \n",
"3-Concert / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 101-Telephone / 272-iPod / \n",
"12-Food / 295-Chocolate / 985-Strawberry / 1403-Flower bouquet / \n",
"6-Animation / 643-Hero / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 2726-Armored Core / \n",
"1-Vehicle / 4-Car / \n",
"7-Musician / 14-Guitar / 63-Acoustic guitar / \n",
"1184-Flag / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"3-Concert / \n",
"591-Naruto: Ultimate Ninja / \n",
"0-Games / 256-Tennis / \n",
"10-Animal / 48-Pet / 132-Bird / 1799-Domestic canary / \n",
"0-Games / 2-Video game / 19-PC game / 725-Human / 739-Guild Wars / 839-Guild Wars 2 / \n",
"125-Television / \n",
"440-Lion / 1039-Tiger / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 302-Call of Duty 4: Modern Warfare / 422-Call of Duty: Ghosts / \n",
"9-Music video / \n",
"10-Animal / 601-Shark / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / 172-Motocross / 197-Trail / 491-Enduro / \n",
"1172-VHS / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 380-Assassin's Creed / 1522-Assassin's Creed II / \n",
"3-Concert / 13-Musical ensemble / 30-Drummer / 39-Drums / 44-Drums / \n",
"10-Animal / 48-Pet / 71-Dog / 695-Balloon / 1030-Trampoline / \n",
"13-Musical ensemble / \n",
"78-Wedding / \n",
"1-Vehicle / 50-Aircraft / 266-Helicopter / 308-Ship / 605-Cruise ship / 2151-Carnival Cruise Lines / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"0-Games / 8-Football / 76-Ball / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"0-Games / 8-Football / 73-Ball / \n",
"93-Comedy / 355-Comedian / \n",
"5-Dance / 1004-Samurai / \n",
"1-Vehicle / 4-Car / 70-Driving / \n",
"5-Dance / 49-School / 95-Talent show / 191-Cheerleading / \n",
"0-Games / 2-Video game / 55-Video game console / 586-Nintendo Entertainment System / \n",
"0-Games / 2-Video game / 426-Resident Evil / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 382-Dota 2 / 549-Defense of the Ancients / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 656-Metal Gear / 2698-Metal Gear Rising: Revengeance / \n",
"390-Model / 1605-Second Life / \n",
"31-Disc jockey / 322-Mixtape / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 30-Drummer / 39-Drums / 44-Drums / \n",
"6-Animation / 15-Cartoon / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"1-Vehicle / 4-Car / 1952-Carburetor / \n",
"0-Games / 159-World of Warcraft / 171-Warcraft / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 81-Athlete / \n",
"0-Games / 2-Video game / 20-Trailer / \n",
"66-Bollywood / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"75-Wrestling / 85-Combat / 117-Boxing / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 234-Ford / 473-Ford Mustang / \n",
"516-Headphones / \n",
"31-Disc jockey / \n",
"466-Gift / \n",
"209-University / \n",
"5-Dance / \n",
"10-Animal / 91-Fish / 288-Underwater / 601-Shark / \n",
"379-Touhou Project / \n",
"0-Games / 2-Video game / 1841-Path of Exile / \n",
"1-Vehicle / 27-Motorcycle / 94-Dashcam / 336-Scooter / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 76-Ball / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 94-Dashcam / 212-Highway / \n",
"0-Games / 211-RuneScape / 1035-Bank / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 832-Headset / \n",
"394-Map / 420-The Elder Scrolls V: Skyrim / \n",
"12-Food / 26-Cooking / 589-Grilling / 620-Barbecue / 1085-Barbecue grill / 1256-Smoking / 2395-Ribs / \n",
"0-Games / 1343-Super Robot Wars / \n",
"3-Concert / 68-Lighting / \n",
"3-Concert / 4180-Alpha / \n",
"444-Family / 2725-Phoenix / \n",
"46-Choir / \n",
"45-Cosmetics / \n",
"28-Fashion / 153-Dress / 390-Model / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 2393-Didgeridoo / \n",
"1035-Bank / 2095-Cash / \n",
"3-Concert / \n",
"61-Art / 148-Painting / 303-Paint / 873-Acrylic paint / 936-Watercolor paint / 1362-Canvas / \n",
"6-Animation / 10-Animal / 15-Cartoon / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 70-Driving / 207-Exhaust system / 680-Subaru / 1041-Subaru Impreza / 1897-Subaru Impreza WRX / \n",
"122-Weight training / \n",
"9-Music video / \n",
"9-Music video / 30-Drummer / \n",
"3-Concert / 7-Musician / \n",
"1-Vehicle / 4-Car / 40-Road / 207-Exhaust system / 299-Volkswagen Passenger Cars / 526-Volkswagen / 531-Hatchback / 668-Volkswagen Golf / 2286-Grand Tourer Injection / 3464-Volkswagen Golf Mk6 / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 220-Mountain bike / 1420-Gear / \n",
"310-The Sims / 328-Simulation video game / 543-The Sims 2 / \n",
"14-Guitar / 100-Electric guitar / \n",
"6-Animation / 9-Music video / \n",
"59-Winter sport / 112-Ice skating / 142-Hockey / 262-Ice / 312-Ice rink / \n",
"1625-Sunset / \n",
"0-Games / 54-Highlight film / 60-Basketball / 168-Basketball moves / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / 556-Skateboarding trick / \n",
"5-Dance / 31-Disc jockey / \n",
"925-Brain / \n",
"25-Toy / 140-LEGO / 884-Lego minifigure / 1302-The Lego Group / 1478-Lego Ninjago / \n",
"0-Games / 2-Video game / 55-Video game console / 199-Wii / 456-Go-kart / 518-Mario Kart / \n",
"6-Animation / 15-Cartoon / 4082-Extra / \n",
"1-Vehicle / 4-Car / 270-Sedan / 403-Toyota / 1583-Toyota Corolla / \n",
"121-Photography / 293-Slide show / \n",
"22-Nature / 61-Art / 97-Drawing / 148-Painting / 303-Paint / 936-Watercolor paint / 3030-Landscape painting / \n",
"7-Musician / 14-Guitar / 24-String instrument / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 1351-Rallycross / \n",
"25-Toy / 179-Kitchen / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / \n",
"45-Cosmetics / 282-Skin / 385-Face / 533-Oil / 2351-Moisturizer / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"6-Animation / 372-Playing card / 460-Yu-Gi-Oh! Trading Card Game / \n",
"5-Dance / 16-Performance art / 911-Studio / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 2907-Suzuka Circuit / \n",
"6-Animation / 15-Cartoon / 616-The Bible / \n",
"0-Games / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / \n",
"0-Games / 25-Toy / 78-Wedding / 200-Doll / 439-Barbie / 1441-Mattel / \n",
"248-Hotel / \n",
"3-Concert / \n",
"0-Games / 1-Vehicle / 25-Toy / 140-LEGO / 1079-Lego City / 4006-Lego Creator / \n",
"587-Clay / 2089-Winnie-the-Pooh / \n",
"1-Vehicle / 62-Train / 103-Rail transport / 119-Locomotive / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 431-Call of Duty: Advanced Warfare / \n",
"0-Games / 2-Video game / 311-Street Fighter / \n",
"3-Concert / 1149-Castle / \n",
"242-Paper / \n",
"0-Games / 2-Video game / 474-Destiny / \n",
"353-Wall / \n",
"5-Dance / \n",
"3-Concert / 9-Music video / \n",
"0-Games / 2-Video game / 6-Animation / 961-Sega Genesis / \n",
"3-Concert / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"67-Cycling / 69-Bicycle / \n",
"12-Food / 56-Hair / 533-Oil / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / \n",
"1-Vehicle / 4-Car / 403-Toyota / 2512-Toyota Prius / \n",
"1-Vehicle / 12-Food / 124-Tractor / 156-Agriculture / 174-Farm / 249-Heavy equipment / 931-Combine Harvester / 1158-Wheat / 3260-Barley / \n",
"1572-Husband / \n",
"9-Music video / 310-The Sims / 543-The Sims 2 / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 135-Drifting / 207-Exhaust system / 344-Coupé / 482-Asphalt / \n",
"12-Food / 86-Plant / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 272-iPod / 331-iPod touch / \n",
"5-Dance / 66-Bollywood / \n",
"2-Video game / 199-Wii / 335-Nintendo 3DS / \n",
"0-Games / 2-Video game / 37-Gadget / 55-Video game console / 199-Wii / 292-Handheld game console / 335-Nintendo 3DS / \n",
"233-Ocean / 1003-Tuba / \n",
"0-Games / 159-World of Warcraft / 171-Warcraft / \n",
"23-Mobile phone / 29-Smartphone / 205-Newscaster / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 169-Rallying / 683-Renault / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / 229-Halo / \n",
"110-Album / \n",
"13-Musical ensemble / 2202-Armed forces / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 1307-Police car / \n",
"7-Musician / 36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"93-Comedy / \n",
"694-Tai chi / \n",
"45-Cosmetics / 238-Nail / 300-Nail art / 304-Nail polish / 307-Hand / 324-Finger / 338-Manicure / 823-Artificial nails / \n",
"14-Guitar / 24-String instrument / \n",
"5-Dance / \n",
"137-Tree / 208-Wood / 673-Saw / 1073-Chainsaw / 1294-Logging / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 219-Accordion / \n",
"23-Mobile phone / 29-Smartphone / 47-Personal computer / 228-Laptop / 497-Macintosh / 513-Computer keyboard / 900-MacBook / 1086-MacBook Pro / 1657-MacBook Air / \n",
"0-Games / 2-Video game / 240-Arcade game / 241-Fighting game / 311-Street Fighter / 461-Street Fighter IV / 496-Super Street Fighter IV / 563-Ryu / \n",
"468-Biology / 1110-Cell / \n",
"0-Games / 59-Winter sport / 112-Ice skating / 142-Hockey / 312-Ice rink / \n",
"3-Concert / 13-Musical ensemble / \n",
"31-Disc jockey / \n",
"0-Games / 8-Football / \n",
"512-Printing / \n",
"3-Concert / \n",
"110-Album / \n",
"5-Dance / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"12-Food / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 184-Vegetable / 210-Cookware and bakeware / 3003-Roll / \n",
"14-Guitar / \n",
"22-Nature / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / 44-Drums / \n",
"762-Angel / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 96-Soldier / 194-Call of Duty: Modern Warfare 2 / 302-Call of Duty 4: Modern Warfare / \n",
"126-The Walt Disney Company / \n",
"33-Weapon / 261-Tool / 315-Knife / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"14-Guitar / 1754-Rocksmith / \n",
"12-Food / \n",
"1-Vehicle / 12-Food / 86-Plant / 124-Tractor / 174-Farm / 459-Plough / 660-Rural area / 906-Sowing / 1129-Hay / 2039-Soybean / \n",
"12-Food / 752-Pasta / 2362-Porcelain / \n",
"125-Television / \n",
"0-Games / 6-Animation / 15-Cartoon / 116-Pokémon / 2597-Sprite / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 25-Toy / 298-Television advertisement / \n",
"102-Building / 318-Church / 870-Chapel / 885-Bell / \n",
"5-Dance / 16-Performance art / \n",
"3-Concert / 8-Football / \n",
"1-Vehicle / 11-Motorsport / 156-Agriculture / 252-Tractor pulling / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / 99-Christmas / \n",
"1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 111-PlayStation 3 / 135-Drifting / 953-Gran Turismo 5 / \n",
"5-Dance / 16-Performance art / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 105-Motorcycling / 669-Extreme sport / \n",
"36-Piano / 53-Keyboard / 107-Musical keyboard / 397-Organ / 1666-Hammond organ / \n",
"0-Games / 1-Vehicle / 11-Motorsport / 21-Stadium / \n",
"1-Vehicle / 18-Outdoor recreation / 42-Fishing / 87-Boat / 466-Gift / 601-Shark / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / 100-Electric guitar / \n",
"28-Fashion / 155-Camera / \n",
"3-Concert / 38-Orchestra / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 422-Call of Duty: Ghosts / \n",
"7-Musician / 13-Musical ensemble / 49-School / 79-American football / 203-Marching band / 306-High school / \n",
"12-Food / 807-Seed / 4566-Chia / \n",
"25-Toy / 428-Rail transport modelling / 507-Thomas the Tank Engine / \n",
"78-Wedding / 153-Dress / 190-Bride / 590-Wedding dress / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 96-Soldier / \n",
"1650-The Idolmaster / 1755-The Idolmaster / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"483-Costume / \n",
"0-Games / 60-Basketball / \n",
"2-Video game / 111-PlayStation 3 / 237-Final Fantasy / 1131-Final Fantasy XIII / \n",
"60-Basketball / 113-House / 255-Slam dunk / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 111-PlayStation 3 / 4507-MotorStorm / \n",
"113-House / 433-Construction / \n",
"0-Games / 2-Video game / 47-Personal computer / 127-Xbox 360 / 181-Xbox / 420-The Elder Scrolls V: Skyrim / \n",
"1-Vehicle / 4-Car / 673-Saw / 1073-Chainsaw / 1294-Logging / 1414-Gasoline / 2209-Firewood / 3073-Log splitter / \n",
"3-Concert / \n",
"0-Games / 2-Video game / 116-Pokémon / 186-Pokémon / \n",
"5-Dance / 16-Performance art / 46-Choir / 209-University / \n",
"0-Games / 8-Football / 73-Ball / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 40-Road / \n",
"3-Concert / \n",
"92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 393-Unmanned aerial vehicle / 3198-Parrot AR.Drone / \n",
"5-Dance / 6-Animation / 1738-Elmo / \n",
"0-Games / 2-Video game / 19-PC game / 241-Fighting game / 3149-Melty Blood / \n",
"425-City / 794-Bridge / 4473-Sydney Opera House / \n",
"10-Animal / 22-Nature / 588-Reptile / 865-Turtle / 2631-Tortoise / \n",
"141-Microsoft Windows / 1938-Rust / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / \n",
"12-Food / \n",
"1-Vehicle / 18-Outdoor recreation / 42-Fishing / 87-Boat / 579-Yacht / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / \n",
"2704-Priston Tale / \n",
"12-Food / 26-Cooking / 620-Barbecue / 2395-Ribs / \n",
"5-Dance / 20-Trailer / \n",
"1-Vehicle / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 197-Trail / 213-Forest / 220-Mountain bike / 284-Mountain biking / 520-Bicycle frame / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / 224-Rapid transit / 821-Rail freight transport / \n",
"9-Music video / \n",
"18-Outdoor recreation / 283-Rock / 458-Gold / 751-Desert / \n",
"8-Football / \n",
"85-Combat / \n",
"163-Running / \n",
"1-Vehicle / 11-Motorsport / 50-Aircraft / 82-Airplane / 129-Aviation / 157-Jet aircraft / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / \n",
"5-Dance / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 24-String instrument / 38-Orchestra / 150-Violin / 450-Viola / \n",
"5-Dance / 95-Talent show / \n",
"9-Music video / 130-Gymnastics / \n",
"14-Guitar / 1533-Lead guitar / \n",
"1-Vehicle / 18-Outdoor recreation / 137-Tree / 1294-Logging / \n",
"6-Animation / 15-Cartoon / 97-Drawing / 183-Manga / \n",
"39-Drums / 44-Drums / \n",
"6-Animation / 15-Cartoon / 123-Naruto / 285-Sasuke Uchiha / 1180-Itachi Uchiha / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 555-Longboard / 667-Longboarding / \n",
"86-Plant / 136-Gardening / 184-Vegetable / 279-Garden / 642-Leaf / 1460-Herb / \n",
"6-Animation / 15-Cartoon / 1445-Donald Duck / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 46-Choir / \n",
"0-Games / 6-Animation / 35-Minecraft / \n",
"0-Games / 159-World of Warcraft / 171-Warcraft / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 41-Sports car / 70-Driving / 231-Supercar / 297-Grand Theft Auto IV / \n",
"66-Bollywood / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 475-Cheese / 542-Potato / \n",
"3-Concert / 7-Musician / \n",
"10-Animal / 48-Pet / \n",
"372-Playing card / 731-Coin / 746-Card manipulation / \n",
"5-Dance / 9-Music video / \n",
"3-Concert / 1044-Coca-Cola / 1224-Cola / \n",
"1-Vehicle / 4-Car / 1461-Electric car / 2304-Electric vehicle / \n",
"9-Music video / 164-Beach / \n",
"3-Concert / 7-Musician / 14-Guitar / 24-String instrument / \n",
"36-Piano / 219-Accordion / \n",
"126-The Walt Disney Company / 144-Amusement park / 250-Parade / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / 482-Asphalt / 556-Skateboarding trick / 667-Longboarding / \n",
"6-Animation / 75-Wrestling / \n",
"6-Animation / \n",
"0-Games / 8-Football / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / \n",
"7-Musician / 13-Musical ensemble / 14-Guitar / \n",
"6-Animation / 15-Cartoon / 89-Comics / 123-Naruto / 183-Manga / 285-Sasuke Uchiha / \n",
"6-Animation / 10-Animal / \n",
"0-Games / 445-Guitar Hero / \n",
"60-Basketball / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 427-Trumpet / \n",
"0-Games / 60-Basketball / 168-Basketball moves / \n",
"12-Food / \n",
"3-Concert / 7-Musician / \n",
"3-Concert / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"162-Medicine / 282-Skin / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"82-Airplane / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / \n",
"5-Dance / \n",
"0-Games / 8-Football / 21-Stadium / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / 44-Drums / \n",
"6-Animation / 15-Cartoon / 116-Pokémon / \n",
"1-Vehicle / 4-Car / 11-Motorsport / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 247-Stallion / 289-Dressage / 438-Mare / \n",
"31-Disc jockey / \n",
"4226-Neo / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 172-Motocross / 491-Enduro / \n",
"8-Football / \n",
"14-Guitar / \n",
"6-Animation / 97-Drawing / 725-Human / \n",
"83-Skateboarding / 108-Skateboard / \n",
"12-Food / 22-Nature / 26-Cooking / 86-Plant / 136-Gardening / 389-Restaurant / \n",
"1-Vehicle / 4-Car / 154-Wheel / 158-Tire / 344-Coupé / 449-Honda / 552-Honda Civic / \n",
"0-Games / 8-Football / 358-Rugby football / \n",
"1-Vehicle / 18-Outdoor recreation / 42-Fishing / 87-Boat / 147-River / \n",
"6-Animation / 9-Music video / \n",
"12-Food / \n",
"67-Cycling / 69-Bicycle / 197-Trail / 220-Mountain bike / 278-GoPro / 284-Mountain biking / \n",
"12-Food / \n",
"162-Medicine / 214-Muscle / \n",
"20-Trailer / 89-Comics / 1173-Captain America / 4345-Winter Soldier / \n",
"209-University / \n",
"1-Vehicle / 4-Car / 245-BMW / 270-Sedan / 2305-BMW 5 Series / 4393-BMW 5 Series (F10) / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 70-Driving / 94-Dashcam / 114-Engine / 270-Sedan / 344-Coupé / 449-Honda / 552-Honda Civic / \n",
"2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 178-Samsung Galaxy / \n",
"0-Games / 90-Sports game / 453-Pro Evolution Soccer / \n",
"0-Games / 2-Video game / 8-Football / 21-Stadium / 90-Sports game / 180-FIFA 15 / \n",
"260-Prayer / 616-The Bible / 971-Alphabet / \n",
"3-Concert / 7-Musician / \n",
"93-Comedy / 1567-Cook / \n",
"28-Fashion / 45-Cosmetics / 238-Nail / 304-Nail polish / \n",
"9-Music video / \n",
"6-Animation / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 47-Personal computer / 72-iPhone / 145-Tablet computer / 178-Samsung Galaxy / 1015-Samsung Galaxy Note series / 1727-Samsung Galaxy Note 3 / \n",
"66-Bollywood / 78-Wedding / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / \n",
"9-Music video / 183-Manga / \n",
"1-Vehicle / 4-Car / 2160-Wagon / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"6-Animation / 15-Cartoon / 477-Transformers / 990-Optimus Prime / 1900-Autobot / 3398-Transformers: Generation 1 / \n",
"448-Planet / 2270-The New York Times / 2643-Asteroid / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"0-Games / 2-Video game / 37-Gadget / 55-Video game console / 116-Pokémon / 186-Pokémon / 292-Handheld game console / 335-Nintendo 3DS / 3191-Mew / 4521-Deoxys / \n",
"12-Food / 215-Eating / \n",
"78-Wedding / 593-Moon / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"0-Games / 2-Video game / 43-Call of Duty / 134-Call of Duty: Black Ops / 138-Call of Duty: Black Ops II / \n",
"0-Games / 2-Video game / 6-Animation / 15-Cartoon / \n",
"23-Mobile phone / 29-Smartphone / \n",
"10-Animal / 80-Horse / 247-Stallion / \n",
"9-Music video / \n",
"8-Football / 205-Newscaster / \n",
"151-Landing / 164-Beach / 447-Sand / 548-Parachuting / 1061-Marriage proposal / \n",
"59-Winter sport / 84-Snow / 120-Winter / 177-Skiing / 267-Mountain / 347-Climbing / \n",
"7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"10-Animal / 80-Horse / 290-Pony / 438-Mare / \n",
"0-Games / 2-Video game / 257-Counter-Strike / \n",
"31-Disc jockey / \n",
"14-Guitar / \n",
"142-Hockey / \n",
"0-Games / 2-Video game / 51-Strategy video game / 165-League of Legends / \n",
"0-Games / 2-Video game / 4409-Infinity Blade / \n",
"14-Guitar / \n",
"6-Animation / 15-Cartoon / 123-Naruto / \n",
"66-Bollywood / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"99-Christmas / \n",
"0-Games / 43-Call of Duty / 138-Call of Duty: Black Ops II / \n",
"110-Album / \n",
"6-Animation / 15-Cartoon / 3853-Akane Tendo / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"96-Soldier / 229-Halo / \n",
"0-Games / 411-MapleStory / \n",
"14-Guitar / 1231-Mandolin / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / \n",
"121-Photography / \n",
"0-Games / 1-Vehicle / 2-Video game / 19-PC game / 286-Bus / \n",
"8-Football / 21-Stadium / \n",
"0-Games / 8-Football / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / 2073-Sabian / \n",
"772-Bowling / \n",
"5-Dance / 9-Music video / \n",
"31-Disc jockey / \n",
"419-Earth / \n",
"1-Vehicle / 4-Car / 74-Truck / 154-Wheel / 158-Tire / 167-Four-wheel drive / 349-Chevrolet / 375-Pickup truck / 1806-GMC / \n",
"0-Games / 772-Bowling / 930-Ten-pin bowling / \n",
"1-Vehicle / 92-Radio-controlled model / 94-Dashcam / 149-Model aircraft / 160-Radio-controlled aircraft / 393-Unmanned aerial vehicle / \n",
"810-Village / \n",
"5-Dance / 244-Goku / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / \n",
"1-Vehicle / 4-Car / 40-Road / 64-Transport / 74-Truck / 94-Dashcam / 482-Asphalt / 1070-Semi-trailer truck / 3220-Dekotora / \n",
"28-Fashion / 45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 291-Rouge / 890-Brush / 3655-Makeup brush / \n",
"61-Art / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 83-Skateboarding / 377-BMX bike / \n",
"5-Dance / 10-Animal / 80-Horse / 247-Stallion / 438-Mare / \n",
"31-Disc jockey / \n",
"5-Dance / 13-Musical ensemble / \n",
"28-Fashion / 483-Costume / \n",
"36-Piano / 53-Keyboard / 61-Art / 97-Drawing / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 169-Rallying / \n",
"6-Animation / \n",
"117-Boxing / 2008-Floyd Mayweather; Jr. vs. Manny Pacquiao / \n",
"1-Vehicle / 4-Car / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 169-Rallying / 198-Off-road vehicle / \n",
"0-Games / 2-Video game / 511-Mortal Kombat / \n",
"319-Robot / \n",
"3-Concert / \n",
"609-Tattoo / \n",
"20-Trailer / \n",
"0-Games / 2-Video game / 735-Kamen Rider Series / \n",
"25-Toy / 200-Doll / \n",
"1-Vehicle / 4-Car / 74-Truck / 154-Wheel / \n",
"8-Football / 31-Disc jockey / 76-Ball / \n",
"9-Music video / 541-Black-and-white / \n",
"6-Animation / 2767-Captain Hook / \n",
"1-Vehicle / 4-Car / 332-Mercedes-Benz / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"93-Comedy / 355-Comedian / \n",
"85-Combat / 1832-Hand-to-hand combat / \n",
"1-Vehicle / 87-Boat / 308-Ship / 579-Yacht / 996-Princess / \n",
"3804-Music of Eritrea / \n",
"9-Music video / \n",
"1-Vehicle / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 197-Trail / 220-Mountain bike / 283-Rock / 284-Mountain biking / \n",
"12-Food / 45-Cosmetics / 56-Hair / 282-Skin / 340-Afro-textured hair / \n",
"6-Animation / 15-Cartoon / \n",
"3-Concert / 68-Lighting / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"3-Concert / \n",
"5-Dance / 28-Fashion / 66-Bollywood / 402-Jewellery / \n",
"3580-Handbell / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 19-PC game / 1837-Realm of the Mad God / \n",
"0-Games / 23-Mobile phone / 29-Smartphone / 72-iPhone / 272-iPod / 331-iPod touch / \n",
"144-Amusement park / 321-Roller coaster / 334-Amusement ride / 3485-PortAventura / \n",
"5-Dance / \n",
"6-Animation / 15-Cartoon / 432-Kingdom Hearts / 665-Kingdom Hearts / 836-Sora / 3299-Kingdom Hearts: Chain of Memories / \n",
"0-Games / 51-Strategy video game / 366-Dragon / 373-Clash of Clans / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"6-Animation / 15-Cartoon / 432-Kingdom Hearts / 665-Kingdom Hearts / 836-Sora / \n",
"92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 393-Unmanned aerial vehicle / 1311-Amazon.com; Inc. / \n",
"0-Games / 2-Video game / 72-iPhone / \n",
"14-Guitar / 63-Acoustic guitar / 670-Banjo / 2241-Plectrum / \n",
"754-Gramophone record / \n",
"1-Vehicle / 4-Car / 114-Engine / \n",
"0-Games / 2-Video game / 60-Basketball / 90-Sports game / 1262-NBA 2K14 / \n",
"3-Concert / 98-Festival / \n",
"3-Concert / 5-Dance / 253-Carnival / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 41-Sports car / 64-Transport / 70-Driving / 94-Dashcam / 212-Highway / 790-Nissan GT-R / \n",
"12-Food / 88-Machine / 640-Bottle / \n",
"20-Trailer / \n",
"5-Dance / \n",
"0-Games / 2-Video game / 55-Video game console / 240-Arcade game / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / 961-Sega Genesis / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 90-Sports game / \n",
"5-Dance / 115-Ballet / \n",
"0-Games / 35-Minecraft / 174-Farm / 1699-Cart / \n",
"125-Television / \n",
"6-Animation / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 38-Orchestra / 39-Drums / \n",
"8-Football / \n",
"261-Tool / 315-Knife / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"102-Building / \n",
"6-Animation / 9-Music video / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 109-Cooking show / 179-Kitchen / 226-Dessert / 232-Cake / 759-Icing / 1088-Infomercial / 1365-Fondant icing / \n",
"60-Basketball / 1399-Streetball / \n",
"2-Video game / \n",
"45-Cosmetics / \n",
"113-House / 1088-Infomercial / 3301-Mop / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / \n",
"0-Games / 2-Video game / 961-Sega Genesis / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 19-PC game / 34-Action-adventure game / 111-PlayStation 3 / 118-Grand Theft Auto V / \n",
"0-Games / 51-Strategy video game / 1687-Heroes of Newerth / \n",
"3-Concert / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 3611-Folding bicycle / \n",
"3-Concert / 7-Musician / 30-Drummer / \n",
"0-Games / 8-Football / 54-Highlight film / 79-American football / 90-Sports game / 561-Madden NFL / 1813-Madden NFL 13 / 2916-Madden NFL 12 / \n",
"56-Hair / \n",
"7-Musician / 9-Music video / 14-Guitar / 412-Monster / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / \n",
"3-Concert / \n",
"1-Vehicle / \n",
"9-Music video / \n",
"6-Animation / 15-Cartoon / \n",
"8-Football / 79-American football / 993-Quarterback / \n",
"3-Concert / 31-Disc jockey / 133-Nightclub / \n",
"0-Games / 2-Video game / 51-Strategy video game / 185-Dragon Ball / 1989-Oculus Rift / \n",
"1-Vehicle / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 197-Trail / 220-Mountain bike / 283-Rock / 284-Mountain biking / \n",
"609-Tattoo / \n",
"0-Games / 2-Video game / 1083-Role-playing game / 3351-Paper Mario: The Thousand-Year Door / 4269-Mario role-playing games / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"0-Games / 10-Animal / 25-Toy / 323-Toddler / 550-Dinosaur / \n",
"93-Comedy / 364-Sitcom / 648-Clown / \n",
"1-Vehicle / 50-Aircraft / 266-Helicopter / 267-Mountain / 1058-Volcano / \n",
"15-Cartoon / 126-The Walt Disney Company / \n",
"47-Personal computer / 228-Laptop / 414-Computer monitor / \n",
"28-Fashion / 368-Sneakers / \n",
"99-Christmas / 370-Origami / 466-Gift / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / \n",
"1-Vehicle / 4-Car / 121-Photography / 155-Camera / 721-Digital SLR / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 608-Wheelie / 4133-Yamaha DT125 / \n",
"325-Vampire / \n",
"5-Dance / 115-Ballet / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 96-Soldier / 134-Call of Duty: Black Ops / \n",
"47-Personal computer / 145-Tablet computer / 242-Paper / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 1097-Touchscreen / \n",
"1202-Academy Awards / \n",
"6-Animation / 15-Cartoon / 183-Manga / \n",
"248-Hotel / \n",
"3-Concert / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"0-Games / 2-Video game / 685-Luigi / 887-King Koopa / 4464-Mario & Luigi: Bowser's Inside Story / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 3901-Star Fox 64 / \n",
"0-Games / 2-Video game / 9-Music video / 19-PC game / 51-Strategy video game / 165-League of Legends / \n",
"5-Dance / 66-Bollywood / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 135-Drifting / 1810-Live for Speed / \n",
"5-Dance / 115-Ballet / \n",
"1-Vehicle / 4-Car / 163-Running / 234-Ford / 1297-Ford Focus / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 132-Bird / 223-Weather / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 1278-Labrador Retriever / 1377-Golden Retriever / 4166-Rescue dog / \n",
"9-Music video / 78-Wedding / 190-Bride / \n",
"4366-Onigiri / \n",
"8-Football / \n",
"3-Concert / 5-Dance / \n",
"14-Guitar / 59-Winter sport / 63-Acoustic guitar / 84-Snow / 177-Skiing / 281-Snowboarding / 305-Ski / 501-Snowboard / 1764-Ski jumping / \n",
"260-Prayer / \n",
"5-Dance / 43-Call of Duty / 102-Building / 302-Call of Duty 4: Modern Warfare / \n",
"309-Computer hardware / 908-Video card / \n",
"42-Fishing / 227-Water / 287-Human swimming / 329-Swimming pool / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 105-Motorcycling / \n",
"9-Music video / \n",
"616-The Bible / \n",
"67-Cycling / 69-Bicycle / 377-BMX bike / \n",
"9-Music video / \n",
"9-Music video / \n",
"45-Cosmetics / 282-Skin / 385-Face / \n",
"3-Concert / \n",
"1-Vehicle / 192-Battlefield / \n",
"2115-Radar / \n",
"1-Vehicle / 18-Outdoor recreation / 22-Nature / 67-Cycling / 69-Bicycle / 86-Plant / 197-Trail / 213-Forest / 220-Mountain bike / 284-Mountain biking / \n",
"3-Concert / 7-Musician / 30-Drummer / 39-Drums / \n",
"1-Vehicle / 74-Truck / 394-Map / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 522-Helmet / 1323-Helmet camera / 1400-rFactor / \n",
"1078-Installation art / \n",
"3-Concert / 68-Lighting / \n",
"28-Fashion / \n",
"8-Football / \n",
"10-Animal / 588-Reptile / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 40-Road / 74-Truck / 121-Photography / 135-Drifting / \n",
"84-Snow / 223-Weather / 1145-Winter storm / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 34-Action-adventure game / 2792-Darksiders / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 40-Road / 57-Race track / \n",
"85-Combat / 937-Black belt / \n",
"28-Fashion / 56-Hair / 106-Hairstyle / 1038-Hijab / \n",
"36-Piano / 53-Keyboard / \n",
"1-Vehicle / 4-Car / \n",
"5-Dance / \n",
"9-Music video / \n",
"754-Gramophone record / \n",
"584-Super Mario Bros. / \n",
"85-Combat / \n",
"6-Animation / 15-Cartoon / 2110-D.Gray-man / \n",
"1-Vehicle / 88-Machine / 261-Tool / 673-Saw / \n",
"0-Games / 2-Video game / 260-Prayer / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 1218-Japanese Cuisine / 1227-Sushi / \n",
"7-Musician / 14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / \n",
"448-Planet / 455-Unidentified flying object / 593-Moon / 1480-Astronaut / \n",
"66-Bollywood / \n",
"0-Games / 54-Highlight film / 60-Basketball / 168-Basketball moves / 322-Mixtape / 1101-Point guard / 3652-3x3 / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 704-Call of Duty: World at War / \n",
"5-Dance / 1443-Dance studio / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 1247-Penalty kick / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"6-Animation / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 101-Telephone / \n",
"35-Minecraft / 131-Computer / \n",
"3-Concert / \n",
"5-Dance / 98-Festival / 250-Parade / 253-Carnival / \n",
"1-Vehicle / 139-Airport / \n",
"31-Disc jockey / \n",
"7-Musician / 13-Musical ensemble / 46-Choir / \n",
"24-String instrument / 150-Violin / \n",
"511-Mortal Kombat / 1679-Scorpion / 1929-Sub-Zero / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"75-Wrestling / 1042-Jumbotron / \n",
"0-Games / 2-Video game / 6-Animation / \n",
"12-Food / 26-Cooking / 32-Recipe / 109-Cooking show / 226-Dessert / 232-Cake / 236-Baking / 354-Egg / 649-Oven / 1322-Biscuit / \n",
"61-Art / 366-Dragon / \n",
"2-Video game / 55-Video game console / 111-PlayStation 3 / 176-PlayStation 4 / \n",
"9-Music video / \n",
"5-Dance / \n",
"22-Nature / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 90-Sports game / \n",
"0-Games / 39-Drums / 44-Drums / 240-Arcade game / 687-Taiko no Tatsujin / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / 272-iPod / 331-iPod touch / \n",
"110-Album / \n",
"3-Concert / \n",
"36-Piano / 46-Choir / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 154-Wheel / 234-Ford / 342-Windows Media Video / 473-Ford Mustang / 647-Muscle car / \n",
"0-Games / 141-Microsoft Windows / \n",
"9-Music video / 2242-Pillow / \n",
"849-Tales / 2195-Tales of Symphonia / \n",
"14-Guitar / 24-String instrument / \n",
"18-Outdoor recreation / 22-Nature / 283-Rock / \n",
"3-Concert / 13-Musical ensemble / \n",
"56-Hair / \n",
"28-Fashion / 1712-Sweater / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"22-Nature / 86-Plant / 136-Gardening / 1500-Cherry blossom / \n",
"10-Animal / 13-Musical ensemble / 38-Orchestra / \n",
"12-Food / 200-Doll / 439-Barbie / 1707-Microwave oven / \n",
"3-Concert / 5-Dance / 7-Musician / 13-Musical ensemble / 16-Performance art / \n",
"0-Games / 19-PC game / 1071-Hearthstone: Heroes of Warcraft / \n",
"9-Music video / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"113-House / 164-Beach / 170-Home improvement / 369-Resort / \n",
"8-Football / 54-Highlight film / \n",
"1-Vehicle / 10-Animal / 18-Outdoor recreation / 217-Hunting / 718-Duck / \n",
"61-Art / 97-Drawing / 408-Sketch / 1432-Kratos / \n",
"5-Dance / 16-Performance art / 115-Ballet / \n",
"0-Games / 599-Dofus / \n",
"3-Concert / 38-Orchestra / \n",
"7-Musician / 14-Guitar / \n",
"1367-Golf club / \n",
"61-Art / 97-Drawing / \n",
"83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"8-Football / \n",
"31-Disc jockey / \n",
"9-Music video / \n",
"213-Forest / 395-Camping / \n",
"0-Games / 2-Video game / 19-PC game / 2277-The Lord of the Rings Online / \n",
"0-Games / 2-Video game / 6-Animation / 20-Trailer / 682-PlayStation Vita / 2628-Sly Cooper / \n",
"0-Games / 6-Animation / 89-Comics / 269-Comic book / 460-Yu-Gi-Oh! Trading Card Game / \n",
"1-Vehicle / 4-Car / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / \n",
"23-Mobile phone / 29-Smartphone / 3850-Weighing scale / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / \n",
"93-Comedy / 1088-Infomercial / \n",
"6-Animation / 1866-Katara / \n",
"20-Trailer / \n",
"0-Games / 6-Animation / 1959-Phantasy Star / \n",
"1-Vehicle / 87-Boat / 2705-Winch / \n",
"0-Games / 2-Video game / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 57-Race track / 456-Go-kart / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 1333-Wangan Midnight / \n",
"2658-Mirror's Edge / \n",
"0-Games / 2-Video game / 55-Video game console / 189-iPad / 387-Spider-Man / 3561-The Amazing Spider-Man / \n",
"0-Games / 2-Video game / 19-PC game / 1938-Rust / \n",
"102-Building / 339-Architecture / 416-Apartment / 939-Condominium / \n",
"130-Gymnastics / \n",
"78-Wedding / 121-Photography / 190-Bride / 293-Slide show / 590-Wedding dress / \n",
"0-Games / 34-Action-adventure game / 35-Minecraft / \n",
"6-Animation / 20-Trailer / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 651-Stock car racing / 728-Dirt track racing / \n",
"0-Games / 60-Basketball / 81-Athlete / 168-Basketball moves / \n",
"1-Vehicle / 10-Animal / 18-Outdoor recreation / 42-Fishing / 87-Boat / 91-Fish / 268-Recreational fishing / 698-Fisherman / \n",
"3-Concert / \n",
"1-Vehicle / 4229-Cable television / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"3-Concert / 7-Musician / 39-Drums / 44-Drums / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 17-Racing / \n",
"0-Games / 51-Strategy video game / 165-League of Legends / \n",
"66-Bollywood / \n",
"1-Vehicle / 4-Car / 751-Desert / 998-Landscape / 1511-Toyota Land Cruiser / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"67-Cycling / 69-Bicycle / 3665-Giant Bicycles / \n",
"3-Concert / 5-Dance / 31-Disc jockey / 68-Lighting / 133-Nightclub / \n",
"6-Animation / 10-Animal / 15-Cartoon / 636-Rabbit / 3090-Bugs Bunny / \n",
"5-Dance / 16-Performance art / \n",
"10-Animal / 18-Outdoor recreation / 91-Fish / 275-Diving / 287-Human swimming / 288-Underwater / 381-Underwater diving / \n",
"539-Home appliance / 818-Washing machine / 863-Body piercing / 1456-Clothes dryer / \n",
"9-Music video / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 40-Road / 57-Race track / 105-Motorcycling / \n",
"15-Cartoon / 123-Naruto / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 1319-Apple / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"164-Beach / 222-Surfing / 815-Surfboard / \n",
"9-Music video / \n",
"5-Dance / \n",
"1465-FOX / \n",
"150-Violin / 401-Cello / 450-Viola / \n",
"99-Christmas / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 207-Exhaust system / 234-Ford / 344-Coupé / 473-Ford Mustang / 1493-Supercharger / 1877-Shelby Mustang / \n",
"1-Vehicle / 259-Fire / 371-Firefighter / \n",
"8-Football / \n",
"25-Toy / 200-Doll / 1842-Dollhouse / \n",
"0-Games / 256-Tennis / 544-Table tennis / \n",
"9-Music video / \n",
"126-The Walt Disney Company / 144-Amusement park / 444-Family / 576-Walt Disney World / \n",
"9-Music video / \n",
"1-Vehicle / 1614-Honda Accord / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 932-Microsoft Lumia / \n",
"20-Trailer / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / \n",
"45-Cosmetics / 111-PlayStation 3 / \n",
"9-Music video / 78-Wedding / \n",
"1-Vehicle / 18-Outdoor recreation / 42-Fishing / 87-Boat / 92-Radio-controlled model / 147-River / 225-Lake / 233-Ocean / 1490-Radio-controlled boat / \n",
"3-Concert / 7-Musician / \n",
"1533-Lead guitar / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / 44-Drums / \n",
"0-Games / 6-Animation / 460-Yu-Gi-Oh! Trading Card Game / \n",
"6-Animation / 15-Cartoon / 801-Inuyasha / \n",
"6-Animation / 10-Animal / 15-Cartoon / 80-Horse / 290-Pony / 343-My Little Pony / \n",
"68-Lighting / 263-Light / 490-Rain / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 43-Call of Duty / 96-Soldier / 138-Call of Duty: Black Ops II / 302-Call of Duty 4: Modern Warfare / 431-Call of Duty: Advanced Warfare / 1311-Amazon.com; Inc. / \n",
"20-Trailer / \n",
"5-Dance / 49-School / 191-Cheerleading / \n",
"3-Concert / \n",
"9-Music video / \n",
"3-Concert / 5-Dance / 31-Disc jockey / \n",
"75-Wrestling / \n",
"18-Outdoor recreation / 163-Running / 674-Marathon / \n",
"9-Music video / \n",
"0-Games / 60-Basketball / 168-Basketball moves / 255-Slam dunk / \n",
"1-Vehicle / 4-Car / 158-Tire / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"3-Concert / \n",
"9-Music video / \n",
"10-Animal / 384-Horse racing / \n",
"56-Hair / \n",
"1026-Foreign Exchange Market / \n",
"1-Vehicle / 4-Car / \n",
"12-Food / 26-Cooking / 32-Recipe / 367-Drink / 613-Bar / 904-Cocktail / 1065-Bartender / 1176-Pumpkin / \n",
"0-Games / 19-PC game / 35-Minecraft / \n",
"5-Dance / 78-Wedding / 190-Bride / 3693-Groomsman / \n",
"6-Animation / 4624-Little Battlers Experience / \n",
"25-Toy / 200-Doll / 412-Monster / 503-Monster High / \n",
"7-Musician / 30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / \n",
"23-Mobile phone / 29-Smartphone / 1032-BlackBerry / \n",
"3-Concert / 5-Dance / 16-Performance art / 133-Nightclub / \n",
"22-Nature / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / 921-Grand Theft Auto: Vice City / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 178-Samsung Galaxy / 554-iPhone 4 / \n",
"1-Vehicle / 266-Helicopter / \n",
"25-Toy / 735-Kamen Rider Series / \n",
"121-Photography / \n",
"1-Vehicle / 4-Car / 41-Sports car / 234-Ford / 1297-Ford Focus / \n",
"9-Music video / 388-Fighter aircraft / \n",
"10-Animal / 18-Outdoor recreation / 80-Horse / 247-Stallion / 404-Livestock / 438-Mare / 2553-Colt / \n",
"246-Furniture / \n",
"59-Winter sport / 84-Snow / 120-Winter / 177-Skiing / 305-Ski / 789-Alpine skiing / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 1124-Yamaha YZF-R1 / \n",
"615-Poker / \n",
"5-Dance / 10-Animal / 78-Wedding / 80-Horse / 98-Festival / 2583-Limousine / \n",
"9-Music video / 13-Musical ensemble / \n",
"6-Animation / 75-Wrestling / 1844-WWE action figures / \n",
"3-Concert / 93-Comedy / \n",
"5-Dance / 440-Lion / \n",
"99-Christmas / 2649-Nativity scene / \n",
"1-Vehicle / 271-Tank / \n",
"364-Sitcom / \n",
"3-Concert / 5-Dance / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 85-Combat / 96-Soldier / 111-PlayStation 3 / 257-Counter-Strike / \n",
"5-Dance / 16-Performance art / \n",
"25-Toy / \n",
"31-Disc jockey / \n",
"0-Games / 9-Music video / 35-Minecraft / \n",
"3-Concert / 38-Orchestra / \n",
"8-Football / 21-Stadium / 54-Highlight film / \n",
"66-Bollywood / \n",
"20-Trailer / \n",
"170-Home improvement / 416-Apartment / \n",
"88-Machine / 812-Plastic / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 378-Drag racing / 473-Ford Mustang / 647-Muscle car / \n",
"489-Board game / 799-Chess / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 516-Headphones / 832-Headset / \n",
"0-Games / 2-Video game / 19-PC game / 43-Call of Duty / 138-Call of Duty: Black Ops II / 422-Call of Duty: Ghosts / \n",
"205-Newscaster / 223-Weather / \n",
"0-Games / 2-Video game / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / 1959-Phantasy Star / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 70-Driving / 231-Supercar / 711-Gran Turismo / 953-Gran Turismo 5 / 1635-Ferrari 458 / \n",
"18-Outdoor recreation / 22-Nature / 147-River / 1195-Rafting / \n",
"3-Concert / 46-Choir / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 67-Cycling / 69-Bicycle / \n",
"3-Concert / \n",
"3-Concert / 16-Performance art / \n",
"5-Dance / \n",
"12-Food / 26-Cooking / 179-Kitchen / \n",
"0-Games / 2-Video game / 125-Television / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 101-Telephone / \n",
"47-Personal computer / 228-Laptop / 465-Microphone / 513-Computer keyboard / \n",
"1082-Knot / 1369-Necktie / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 229-Halo / 522-Helmet / \n",
"1-Vehicle / 64-Transport / 286-Bus / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 57-Race track / 59-Winter sport / 84-Snow / 120-Winter / 610-Snowmobile / \n",
"8-Football / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 1116-Goat / 3147-Goat Simulator / \n",
"9-Music video / 22-Nature / 213-Forest / \n",
"36-Piano / \n",
"9-Music video / \n",
"3-Concert / 5-Dance / 98-Festival / 126-The Walt Disney Company / 144-Amusement park / 250-Parade / 576-Walt Disney World / 679-Mickey Mouse / \n",
"9-Music video / \n",
"209-University / \n",
"8-Football / 623-Coach / \n",
"6-Animation / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / \n",
"0-Games / 35-Minecraft / 425-City / \n",
"67-Cycling / 69-Bicycle / 83-Skateboarding / 108-Skateboard / 221-Skatepark / 336-Scooter / 377-BMX bike / \n",
"1-Vehicle / 68-Lighting / 263-Light / 744-Window / 1020-Emergency vehicle / 3070-Emergency vehicle lighting / \n",
"25-Toy / 626-Puzzle / 631-Cube / 740-Rubik's Cube / \n",
"0-Games / 1-Vehicle / 2-Video game / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 172-Motocross / 2930-MX vs. ATV / \n",
"5-Dance / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"28-Fashion / 66-Bollywood / \n",
"1726-Harlem Shake (meme) / \n",
"102-Building / \n",
"1-Vehicle / \n",
"3-Concert / \n",
"12-Food / 720-Wine / \n",
"10-Animal / 91-Fish / \n",
"842-Harp / \n",
"56-Hair / 106-Hairstyle / 1089-Wire / 2697-Zebra / \n",
"12-Food / 26-Cooking / 32-Recipe / 454-Sauce / 671-Tomato / 752-Pasta / \n",
"707-Roof / \n",
"915-Town / \n",
"3-Concert / 5-Dance / 14-Guitar / 16-Performance art / 361-Flamenco / \n",
"12-Food / 232-Cake / \n",
"0-Games / 21-Stadium / 75-Wrestling / 77-Arena / \n",
"61-Art / 479-Graffiti / 911-Studio / \n",
"3-Concert / 7-Musician / 125-Television / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"3-Concert / 16-Performance art / \n",
"0-Games / 2-Video game / 34-Action-adventure game / \n",
"2277-The Lord of the Rings Online / 2759-Lute / \n",
"31-Disc jockey / \n",
"25-Toy / 140-LEGO / 290-Pony / 343-My Little Pony / 524-Littlest Pet Shop / 1192-Hello Kitty / 3899-Lego Minecraft / \n",
"2-Video game / 47-Personal computer / 1028-Devil May Cry / 1893-Devil May Cry 4 / \n",
"3-Concert / 5-Dance / 95-Talent show / \n",
"10-Animal / \n",
"1016-Glove / \n",
"1-Vehicle / 62-Train / 64-Transport / \n",
"9-Music video / \n",
"23-Mobile phone / 86-Plant / 155-Camera / 1117-Camcorder / \n",
"0-Games / 1138-Tower of Saviors / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 378-Drag racing / 935-Off-road racing / 1699-Cart / 4003-Baja 1000 / \n",
"20-Trailer / \n",
"130-Gymnastics / 1451-Ribbon / \n",
"5-Dance / 95-Talent show / 130-Gymnastics / 1835-Majorette / \n",
"0-Games / 8-Football / 65-Kick / 76-Ball / \n",
"6-Animation / 15-Cartoon / 116-Pokémon / 4379-Team Rocket / \n",
"0-Games / 2-Video game / 19-PC game / 127-Xbox 360 / 1196-Dragon Age / 2179-Dragon Age: Origins / \n",
"10-Animal / 80-Horse / 438-Mare / \n",
"22-Nature / 36-Piano / 53-Keyboard / 1500-Cherry blossom / \n",
"75-Wrestling / 85-Combat / \n",
"3789-Circular saw / \n",
"9-Music video / \n",
"222-Surfing / 329-Swimming pool / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 235-Lipstick / 291-Rouge / 436-Concealer / \n",
"205-Newscaster / 913-President / \n",
"1-Vehicle / 4-Car / 3762-PSA HDi engine / \n",
"350-Circus / 788-Juggling / 1100-Hat / \n",
"67-Cycling / 69-Bicycle / 220-Mountain bike / 284-Mountain biking / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"56-Hair / 106-Hairstyle / 340-Afro-textured hair / 1269-Human hair color / \n",
"6-Animation / 1809-Tokyo Mew Mew / \n",
"248-Hotel / \n",
"5-Dance / 9-Music video / 133-Nightclub / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 127-Xbox 360 / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / 2028-Sonic World Adventure / \n",
"0-Games / 2-Video game / \n",
"83-Skateboarding / 108-Skateboard / \n",
"20-Trailer / \n",
"6-Animation / 15-Cartoon / 396-One Piece / \n",
"0-Games / 2-Video game / 55-Video game console / 199-Wii / 3325-Super Paper Mario / 4193-Pit / \n",
"16-Performance art / 31-Disc jockey / \n",
"208-Wood / 943-Stove / 2707-Wood-burning stove / \n",
"253-Carnival / \n",
"66-Bollywood / \n",
"1-Vehicle / 25-Toy / 62-Train / 119-Locomotive / 140-LEGO / 1079-Lego City / \n",
"3461-Noah's Ark / \n",
"9-Music video / \n",
"14-Guitar / \n",
"12-Food / 86-Plant / 136-Gardening / 671-Tomato / \n",
"1-Vehicle / 4-Car / 1414-Gasoline / 2616-Chevrolet S-10 / \n",
"117-Boxing / \n",
"0-Games / 489-Board game / 922-Tabletop game / \n",
"45-Cosmetics / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 235-Lipstick / 1012-Lip gloss / \n",
"361-Flamenco / \n",
"9-Music video / \n",
"93-Comedy / \n",
"36-Piano / \n",
"0-Games / 489-Board game / \n",
"209-University / 1246-Campus / \n",
"3-Concert / 38-Orchestra / \n",
"28-Fashion / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / \n",
"10-Animal / 48-Pet / 1147-Hamster / \n",
"0-Games / 2-Video game / 43-Call of Duty / 134-Call of Duty: Black Ops / 194-Call of Duty: Modern Warfare 2 / \n",
"295-Chocolate / \n",
"0-Games / 2-Video game / 51-Strategy video game / 382-Dota 2 / 549-Defense of the Ancients / \n",
"9-Music video / \n",
"162-Medicine / \n",
"8-Football / 21-Stadium / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 195-Mascara / 196-Eye / 218-Eyelash / 235-Lipstick / 291-Rouge / \n",
"1289-Caporales / \n",
"7-Musician / 9-Music video / 14-Guitar / \n",
"9-Music video / \n",
"23-Mobile phone / 29-Smartphone / 1820-Motorola Droid / \n",
"208-Wood / 492-Floor / 1179-Brick / 1869-Cement / \n",
"1-Vehicle / 50-Aircraft / 129-Aviation / 139-Airport / 151-Landing / 175-Airliner / 187-Takeoff / 188-Airline / 254-Runway / 923-Boeing 777 / \n",
"59-Winter sport / 112-Ice skating / 262-Ice / 312-Ice rink / 316-Figure skating / 632-Ice dancing / 833-Ice skate / \n",
"1-Vehicle / \n",
"12-Food / 612-Beer / \n",
"93-Comedy / 341-Sketch comedy / 2764-Mad / \n",
"0-Games / 60-Basketball / 168-Basketball moves / 255-Slam dunk / 1101-Point guard / \n",
"6-Animation / 15-Cartoon / \n",
"0-Games / 59-Winter sport / 112-Ice skating / 142-Hockey / 312-Ice rink / 342-Windows Media Video / \n",
"31-Disc jockey / \n",
"5-Dance / \n",
"8-Football / 21-Stadium / \n",
"0-Games / 488-Door / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 24-String instrument / 38-Orchestra / 1081-Lemon / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 982-V8 engine / 1466-Pontiac / \n",
"6-Animation / 310-The Sims / 328-Simulation video game / 564-The Sims 3 / \n",
"49-School / 410-Teacher / 845-Classroom / \n",
"10-Animal / 80-Horse / 247-Stallion / 289-Dressage / 351-Jumping / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 2283-Beagle / \n",
"0-Games / 2-Video game / 199-Wii / 518-Mario Kart / 887-King Koopa / \n",
"507-Thomas the Tank Engine / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / 224-Rapid transit / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 40-Road / 297-Grand Theft Auto IV / 658-Grand Theft Auto: The Lost and Damned / \n",
"10-Animal / 48-Pet / 166-Cat / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 73-Ball / 79-American football / 81-Athlete / 358-Rugby football / \n",
"31-Disc jockey / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / \n",
"341-Sketch comedy / 364-Sitcom / \n",
"6-Animation / 15-Cartoon / 1946-Total Drama Island / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 105-Motorcycling / 172-Motocross / 197-Trail / 491-Enduro / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 449-Honda / 2223-Honda S2000 / \n",
"9-Music video / \n",
"1-Vehicle / 124-Tractor / 156-Agriculture / 174-Farm / 715-Farming Simulator / \n",
"3-Concert / 7-Musician / \n",
"0-Games / 394-Map / 910-Dragon Quest / 1426-Treasure / \n",
"1-Vehicle / 4-Car / 345-Loudspeaker / 484-Subwoofer / \n",
"49-School / 2191-Text messaging / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 38-Orchestra / 44-Drums / \n",
"230-Brass instrument / 296-Saxophone / 1006-Tenor saxophone / \n",
"1-Vehicle / 50-Aircraft / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 266-Helicopter / 392-Radio-controlled helicopter / \n",
"7-Musician / 14-Guitar / 100-Electric guitar / \n",
"3-Concert / 68-Lighting / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 220-Mountain bike / 284-Mountain biking / 520-Bicycle frame / \n",
"10-Animal / 91-Fish / 258-Aquarium / \n",
"7-Musician / 9-Music video / 30-Drummer / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"125-Television / 1594-Nerd / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 42-Fishing / 91-Fish / 288-Underwater / 481-Kayak / \n",
"0-Games / 2-Video game / 90-Sports game / 180-FIFA 15 / \n",
"28-Fashion / 56-Hair / 106-Hairstyle / 340-Afro-textured hair / 583-Wig / 857-Afro / \n",
"20-Trailer / 294-Batman / 2146-Bane / \n",
"36-Piano / 53-Keyboard / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / \n",
"0-Games / 2-Video game / 55-Video game console / \n",
"86-Plant / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 165-League of Legends / 1494-Jungle / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 210-Cookware and bakeware / 239-Roasting / 688-Beef / 1332-Onion / \n",
"25-Toy / 99-Christmas / 323-Toddler / 466-Gift / \n",
"1088-Infomercial / \n",
"6-Animation / 25-Toy / 435-Action figure / 477-Transformers / 4445-Transformers: Beast Wars / \n",
"0-Games / 2-Video game / 872-Diablo / 2054-Diablo II / 3509-Diablo II: Lord of Destruction / \n",
"0-Games / 2-Video game / 257-Counter-Strike / 702-Counter-Strike: Source / \n",
"88-Machine / 666-Concrete / 1678-Foam / \n",
"1-Vehicle / 4-Car / 74-Truck / \n",
"3-Concert / \n",
"5-Dance / \n",
"1-Vehicle / 4-Car / \n",
"12-Food / 32-Recipe / 226-Dessert / 232-Cake / 236-Baking / 295-Chocolate / 437-Cream / 560-Sugar / 578-Cookie / 600-Flour / 759-Icing / 795-Cupcake / 1102-Cake decorating / 1365-Fondant icing / 3087-Sugar paste / \n",
"0-Games / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / \n",
"18-Outdoor recreation / 573-Paragliding / 778-Gliding / 1602-Hang gliding / \n",
"8-Football / \n",
"0-Games / 35-Minecraft / \n",
"45-Cosmetics / 173-Eye shadow / 193-Eye liner / 196-Eye / 291-Rouge / \n",
"5-Dance / 16-Performance art / \n",
"0-Games / 51-Strategy video game / 373-Clash of Clans / \n",
"6-Animation / 15-Cartoon / 185-Dragon Ball / 244-Goku / 4618-Android 17 / \n",
"191-Cheerleading / \n",
"14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"0-Games / 6-Animation / 15-Cartoon / \n",
"3-Concert / \n",
"6-Animation / 15-Cartoon / 183-Manga / \n",
"372-Playing card / 746-Card manipulation / \n",
"3-Concert / \n",
"9-Music video / \n",
"0-Games / 2-Video game / 89-Comics / 185-Dragon Ball / \n",
"5-Dance / 79-American football / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 264-Star Wars / \n",
"318-Church / \n",
"248-Hotel / 369-Resort / \n",
"28-Fashion / 45-Cosmetics / 469-Bag / \n",
"708-Mermaid / 981-Sculpture / 2456-Tail / \n",
"6-Animation / 15-Cartoon / 20-Trailer / \n",
"6-Animation / 2544-Megami Tensei / 3774-Shin Megami Tensei / \n",
"3-Concert / 5-Dance / 49-School / 95-Talent show / 191-Cheerleading / 595-Champion / \n",
"1-Vehicle / 4-Car / 299-Volkswagen Passenger Cars / 4067-Škoda Fabia / \n",
"85-Combat / 117-Boxing / \n",
"121-Photography / 633-Photograph / \n",
"35-Minecraft / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 40-Road / 105-Motorcycling / 172-Motocross / 608-Wheelie / 927-Supermoto / 2975-BMW Motorrad / \n",
"0-Games / 51-Strategy video game / 165-League of Legends / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 74-Truck / 94-Dashcam / 212-Highway / \n",
"372-Playing card / 746-Card manipulation / \n",
"12-Food / 26-Cooking / 32-Recipe / 109-Cooking show / 179-Kitchen / 442-Bread / 475-Cheese / 542-Potato / 3003-Roll / \n",
"1-Vehicle / 9-Music video / \n",
"6-Animation / 15-Cartoon / 185-Dragon Ball / 244-Goku / \n",
"7-Musician / 78-Wedding / \n",
"31-Disc jockey / \n",
"12-Food / 26-Cooking / 52-Dish / 58-Cuisine / 215-Eating / 301-Meal / 389-Restaurant / 1949-Octopus / \n",
"5-Dance / \n",
"6-Animation / 15-Cartoon / 320-Super Smash Bros. / \n",
"5-Dance / 20-Trailer / \n",
"96-Soldier / 736-ARMA / \n",
"0-Games / 2-Video game / 19-PC game / 96-Soldier / 192-Battlefield / 374-Battlefield 3 / \n",
"1-Vehicle / 4-Car / \n",
"261-Tool / \n",
"0-Games / 6-Animation / 15-Cartoon / 547-Bleach / 3562-Hollow / 3744-Tōshirō Hitsugaya / 4642-Kenpachi Zaraki / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / \n",
"8-Football / 21-Stadium / \n",
"6-Animation / 113-House / 246-Furniture / 416-Apartment / \n",
"1-Vehicle / 4-Car / 234-Ford / 2514-Seat / \n",
"171-Warcraft / \n",
"9-Music video / \n",
"6-Animation / 15-Cartoon / 624-Number / \n",
"93-Comedy / \n",
"0-Games / 2-Video game / 159-World of Warcraft / 1356-Paladin / \n",
"8-Football / \n",
"3-Concert / 68-Lighting / \n",
"1-Vehicle / 2-Video game / 4-Car / 70-Driving / 111-PlayStation 3 / 127-Xbox 360 / 352-Need for Speed / 2820-Need For Speed: ProStreet / \n",
"1510-Liquid / \n",
"5-Dance / 16-Performance art / \n",
"323-Toddler / 783-Zee TV / \n",
"376-Flute / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 109-Cooking show / 210-Cookware and bakeware / 239-Roasting / 274-Meat / \n",
"3-Concert / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / \n",
"0-Games / 2-Video game / 19-PC game / 34-Action-adventure game / 656-Metal Gear / 3284-Metal Gear Solid 2: Sons of Liberty / \n",
"0-Games / 2-Video game / 237-Final Fantasy / \n",
"28-Fashion / \n",
"3-Concert / \n",
"3-Concert / \n",
"8-Football / \n",
"0-Games / 35-Minecraft / \n",
"78-Wedding / 1427-TalesRunner / \n",
"7-Musician / 30-Drummer / 39-Drums / 44-Drums / 584-Super Mario Bros. / \n",
"10-Animal / 25-Toy / 550-Dinosaur / 1126-Tyrannosaurus / \n",
"5-Dance / 49-School / 95-Talent show / 191-Cheerleading / \n",
"6-Animation / 547-Bleach / \n",
"3-Concert / 7-Musician / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 167-Four-wheel drive / 197-Trail / 198-Off-road vehicle / 327-Mud / 360-Jeep / \n",
"1-Vehicle / 4-Car / 299-Volkswagen Passenger Cars / 526-Volkswagen / 969-Volkswagen Beetle / \n",
"3063-Friendship bracelet / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / 1603-Fox / \n",
"78-Wedding / 1061-Marriage proposal / \n",
"75-Wrestling / \n",
"14-Guitar / 24-String instrument / 670-Banjo / \n",
"0-Games / 90-Sports game / 199-Wii / 3171-Wii Sports / \n",
"4235-American Kenpo / \n",
"162-Medicine / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 378-Drag racing / \n",
"36-Piano / 53-Keyboard / \n",
"10-Animal / 166-Cat / \n",
"1-Vehicle / 4-Car / 11-Motorsport / \n",
"61-Art / 102-Building / \n",
"12-Food / 469-Bag / 500-Rice / \n",
"6-Animation / 15-Cartoon / \n",
"18-Outdoor recreation / 59-Winter sport / 177-Skiing / 305-Ski / \n",
"0-Games / 1-Vehicle / 4-Car / 34-Action-adventure game / 297-Grand Theft Auto IV / 1661-Volkswagen Gol / \n",
"8-Football / \n",
"0-Games / 2-Video game / 19-PC game / 111-PlayStation 3 / 127-Xbox 360 / 1007-Portal / 1274-Portal 2 / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 215-Eating / 301-Meal / 662-Pizza / 1176-Pumpkin / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"988-Ibiza / \n",
"5-Dance / \n",
"12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / 109-Cooking show / 210-Cookware and bakeware / 274-Meat / 2313-Bánh / \n",
"419-Earth / 712-Outer space / 1480-Astronaut / \n",
"3-Concert / 5-Dance / 16-Performance art / 68-Lighting / \n",
"3-Concert / 68-Lighting / \n",
"75-Wrestling / 85-Combat / \n",
"18-Outdoor recreation / 22-Nature / 217-Hunting / 1215-Wild boar / \n",
"110-Album / \n",
"38-Orchestra / \n",
"141-Microsoft Windows / \n",
"31-Disc jockey / \n",
"17-Racing / 769-Red Bull / \n",
"5-Dance / 130-Gymnastics / \n",
"22-Nature / 260-Prayer / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 36-Piano / 38-Orchestra / 104-Pianist / \n",
"0-Games / 411-MapleStory / \n",
"1-Vehicle / 18-Outdoor recreation / 67-Cycling / 69-Bicycle / 220-Mountain bike / 284-Mountain biking / 570-Road bicycle racing / \n",
"25-Toy / 200-Doll / \n",
"0-Games / 2-Video game / 89-Comics / 387-Spider-Man / 748-Iron Man / 855-Hulk / 897-Avengers / 1173-Captain America / 1982-Venom / 2172-Deadpool / 4401-Iron Man's armor / 4554-Marvel Super Hero Squad Online / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 352-Need for Speed / 647-Muscle car / 761-Chevrolet Camaro / 2772-Shift 2: Unleashed / \n",
"88-Machine / 319-Robot / 372-Playing card / \n",
"0-Games / 8-Football / 90-Sports game / 180-FIFA 15 / \n",
"83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"3-Concert / 7-Musician / 46-Choir / \n",
"8-Football / \n",
"0-Games / 51-Strategy video game / 2990-Command & Conquer: Red Alert / 3430-Command & Conquer: Red Alert 3 / \n",
"2-Video game / 20-Trailer / 34-Action-adventure game / 1144-Castlevania / 3768-Castlevania: Lords of Shadow / \n",
"2-Video game / 20-Trailer / 55-Video game console / 199-Wii / 320-Super Smash Bros. / 335-Nintendo 3DS / 443-Wii U / 697-Super Smash Bros. for Nintendo 3DS and Wii U / 2750-Mii / \n",
"3-Concert / \n",
"5-Dance / 9-Music video / \n",
"0-Games / 25-Toy / 455-Unidentified flying object / 596-Monster Hunter / 1668-Claw crane / \n",
"8-Football / \n",
"0-Games / 2-Video game / 19-PC game / 159-World of Warcraft / 171-Warcraft / \n",
"1442-Tabla / \n",
"59-Winter sport / 84-Snow / 177-Skiing / 278-GoPro / 281-Snowboarding / \n",
"46-Choir / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 25-Toy / 92-Radio-controlled model / 135-Drifting / 202-Radio-controlled car / 1414-Gasoline / 2753-SEAT León / \n",
"31-Disc jockey / \n",
"1-Vehicle / 4-Car / 40-Road / 94-Dashcam / 800-Mountain pass / \n",
"0-Games / 1680-Seafight / \n",
"0-Games / 51-Strategy video game / 165-League of Legends / \n",
"1-Vehicle / 50-Aircraft / 149-Model aircraft / 471-Wing / \n",
"12-Food / 640-Bottle / 720-Wine / \n",
"3-Concert / 7-Musician / 14-Guitar / 24-String instrument / \n",
"162-Medicine / \n",
"1-Vehicle / 4-Car / 124-Tractor / 249-Heavy equipment / 363-Soil / 817-Bulldozer / \n",
"59-Winter sport / 112-Ice skating / 115-Ballet / 262-Ice / 312-Ice rink / 316-Figure skating / 632-Ice dancing / 833-Ice skate / \n",
"6-Animation / 25-Toy / 200-Doll / 439-Barbie / \n",
"0-Games / 2-Video game / 25-Toy / 140-LEGO / 264-Star Wars / 714-Jedi / 717-Darth Vader / 852-Lego Star Wars / 1393-Lightsaber / 4117-Leia Organa / \n",
"12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / 295-Chocolate / 1765-Muffin / 2699-Chocolate chip / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"0-Games / 2-Video game / 47-Personal computer / 131-Computer / 586-Nintendo Entertainment System / \n",
"9-Music video / \n",
"12-Food / 26-Cooking / 32-Recipe / 52-Dish / 58-Cuisine / 391-Chicken meat / 424-Chicken / 1509-Stew / \n",
"0-Games / 8-Football / 54-Highlight film / 358-Rugby football / \n",
"60-Basketball / 255-Slam dunk / \n",
"0-Games / 8-Football / 54-Highlight film / \n",
"5-Dance / 2198-Irish dance / \n",
"7-Musician / \n",
"9-Music video / \n",
"3-Concert / 7-Musician / 14-Guitar / \n",
"8-Football / \n",
"125-Television / \n",
"1-Vehicle / \n",
"28-Fashion / 206-Shoe / 1800-Football boot / \n",
"6-Animation / 10-Animal / 2648-Larva / \n",
"6-Animation / 1934-Hunter × Hunter / \n",
"5-Dance / 49-School / \n",
"20-Trailer / \n",
"7-Musician / 14-Guitar / 24-String instrument / 150-Violin / 639-Fiddle / \n",
"3-Concert / 7-Musician / 16-Performance art / 30-Drummer / \n",
"0-Games / 35-Minecraft / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / \n",
"0-Games / 51-Strategy video game / \n",
"33-Weapon / 362-PlayStation Portable / \n",
"5-Dance / 115-Ballet / 889-Swing / \n",
"0-Games / 21-Stadium / 1182-Softball / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 169-Rallying / \n",
"75-Wrestling / \n",
"6-Animation / \n",
"1-Vehicle / 62-Train / 64-Transport / 103-Rail transport / 119-Locomotive / 143-Railroad car / 152-Track / 161-Train station / \n",
"12-Food / 86-Plant / 136-Gardening / 156-Agriculture / 542-Potato / \n",
"237-Final Fantasy / 3069-Final Fantasy V / \n",
"12-Food / 26-Cooking / 32-Recipe / \n",
"10-Animal / 18-Outdoor recreation / 22-Nature / 42-Fishing / 91-Fish / 275-Diving / 287-Human swimming / 288-Underwater / 407-Scuba diving / \n",
"28-Fashion / 153-Dress / \n",
"7-Musician / 38-Orchestra / 230-Brass instrument / \n",
"0-Games / \n",
"5-Dance / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 81-Athlete / \n",
"0-Games / 2-Video game / 25-Toy / 140-LEGO / 264-Star Wars / 852-Lego Star Wars / 4356-R2-D2 / \n",
"14-Guitar / 269-Comic book / \n",
"125-Television / \n",
"1-Vehicle / 4-Car / 40-Road / 70-Driving / 94-Dashcam / 663-Street / 1285-Parking / \n",
"2-Video game / 23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / 178-Samsung Galaxy / 899-Samsung Galaxy S III / 1773-Samsung Galaxy S series / \n",
"5-Dance / \n",
"1-Vehicle / 286-Bus / \n",
"3-Concert / 5-Dance / 16-Performance art / 68-Lighting / \n",
"9-Music video / \n",
"93-Comedy / \n",
"6-Animation / 2237-Kantai Collection / \n",
"0-Games / 8-Football / 90-Sports game / 180-FIFA 15 / \n",
"28-Fashion / 59-Winter sport / 112-Ice skating / 142-Hockey / \n",
"0-Games / 5-Dance / 49-School / 79-American football / 191-Cheerleading / \n",
"1-Vehicle / 4-Car / \n",
"10-Animal / 1177-Spider / \n",
"0-Games / 8-Football / 21-Stadium / 73-Ball / 77-Arena / \n",
"1-Vehicle / 4-Car / 509-Classic car / 1455-Cadillac / \n",
"358-Rugby football / \n",
"23-Mobile phone / 29-Smartphone / 125-Television / \n",
"0-Games / 4162-Tales of Pirates / \n",
"83-Skateboarding / 108-Skateboard / 433-Construction / 2866-Half-pipe / \n",
"0-Games / 2-Video game / 55-Video game console / 199-Wii / 456-Go-kart / 518-Mario Kart / \n",
"93-Comedy / \n",
"0-Games / 8-Football / 21-Stadium / 73-Ball / 76-Ball / \n",
"5-Dance / \n",
"1-Vehicle / 11-Motorsport / 27-Motorcycle / 105-Motorcycling / 114-Engine / 1036-Moped / \n",
"25-Toy / 262-Ice / 323-Toddler / 417-Play-Doh / 437-Cream / 684-Ice cream / \n",
"89-Comics / 123-Naruto / \n",
"88-Machine / 400-Woodturning / 1043-Router / \n",
"0-Games / 2-Video game / 19-PC game / 35-Minecraft / \n",
"12-Food / \n",
"7-Musician / 13-Musical ensemble / 44-Drums / 46-Choir / 820-Bagpipes / 1646-Pipe band / \n",
"0-Games / 33-Weapon / 834-Combat Arms / \n",
"83-Skateboarding / 108-Skateboard / 555-Longboard / 667-Longboarding / \n",
"511-Mortal Kombat / \n",
"18-Outdoor recreation / 59-Winter sport / 84-Snow / 120-Winter / 177-Skiing / 281-Snowboarding / 305-Ski / 2921-Snowkiting / \n",
"3-Concert / 7-Musician / \n",
"10-Animal / 48-Pet / 71-Dog / 2283-Beagle / \n",
"7-Musician / 886-James Bond / \n",
"9-Music video / \n",
"1-Vehicle / 4-Car / 864-Speedometer / 2016-Volvo Cars / \n",
"1-Vehicle / 4-Car / 74-Truck / 2701-Tow truck / \n",
"364-Sitcom / \n",
"9-Music video / \n",
"0-Games / 60-Basketball / \n",
"0-Games / 2-Video game / 14-Guitar / 445-Guitar Hero / 724-Rock Band / 1703-Rock Band / \n",
"0-Games / 372-Playing card / 494-Card game / 746-Card manipulation / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 44-Drums / 689-Harmonica / 2922-Richter-tuned harmonica / \n",
"1-Vehicle / 10-Animal / 18-Outdoor recreation / 42-Fishing / 147-River / \n",
"15-Cartoon / 648-Clown / 3696-Ronald McDonald / \n",
"10-Animal / 22-Nature / 132-Bird / \n",
"3816-Indie film / \n",
"9-Music video / \n",
"31-Disc jockey / \n",
"99-Christmas / 558-Santa Claus / \n",
"102-Building / 1733-Ant / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 30-Drummer / 44-Drums / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 176-PlayStation 4 / 294-Batman / 838-Batman: Arkham / 1881-Arkham Asylum / \n",
"13-Musical ensemble / 203-Marching band / 250-Parade / \n",
"31-Disc jockey / \n",
"2-Video game / 31-Disc jockey / 68-Lighting / \n",
"0-Games / 2-Video game / 90-Sports game / \n",
"28-Fashion / \n",
"0-Games / 75-Wrestling / 85-Combat / 90-Sports game / 3764-EA Sports UFC / \n",
"22-Nature / \n",
"12-Food / 273-News program / \n",
"3603-Quartz / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"1119-Sensor / \n",
"3-Concert / 16-Performance art / \n",
"8-Football / 79-American football / \n",
"286-Bus / \n",
"12-Food / 32-Recipe / 162-Medicine / 786-Juice / \n",
"7-Musician / 30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"7-Musician / 14-Guitar / \n",
"0-Games / 8-Football / \n",
"3-Concert / \n",
"75-Wrestling / \n",
"0-Games / 1211-Transformice / \n",
"0-Games / 8-Football / 54-Highlight film / 65-Kick / 73-Ball / 90-Sports game / 180-FIFA 15 / \n",
"6-Animation / 15-Cartoon / 61-Art / \n",
"2-Video game / 6-Animation / 20-Trailer / 1523-Alice / \n",
"0-Games / 2-Video game / 43-Call of Duty / 96-Soldier / 138-Call of Duty: Black Ops II / \n",
"296-Saxophone / \n",
"9-Music video / \n",
"121-Photography / \n",
"6-Animation / 89-Comics / 803-DC Comics / \n",
"6-Animation / 10-Animal / 15-Cartoon / 126-The Walt Disney Company / 440-Lion / 1156-Simba / 4202-Timon and Pumbaa / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / \n",
"7-Musician / 9-Music video / 14-Guitar / \n",
"28-Fashion / 469-Bag / 676-Handbag / 1163-Carpet / \n",
"5-Dance / 115-Ballet / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 133-Nightclub / \n",
"59-Winter sport / 112-Ice skating / 142-Hockey / \n",
"260-Prayer / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / \n",
"1-Vehicle / 4-Car / 74-Truck / 158-Tire / 167-Four-wheel drive / 375-Pickup truck / 3541-Nissan Navara / \n",
"1-Vehicle / 10-Animal / 124-Tractor / 156-Agriculture / \n",
"66-Bollywood / \n",
"5-Dance / 110-Album / \n",
"89-Comics / 803-DC Comics / \n",
"6-Animation / 25-Toy / 524-Littlest Pet Shop / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"45-Cosmetics / 162-Medicine / \n",
"0-Games / 6-Animation / 9-Music video / 35-Minecraft / \n",
"0-Games / 2-Video game / 6-Animation / 116-Pokémon / 186-Pokémon / 320-Super Smash Bros. / \n",
"28-Fashion / 206-Shoe / 757-Boot / \n",
"28-Fashion / 956-Glasses / 1481-Sunglasses / \n",
"6-Animation / 15-Cartoon / \n",
"6-Animation / 15-Cartoon / \n",
"9-Music video / \n",
"209-University / 293-Slide show / \n",
"7-Musician / 14-Guitar / 63-Acoustic guitar / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 527-Battery / 1159-Battery charger / \n",
"1-Vehicle / 11-Motorsport / 154-Wheel / 158-Tire / 935-Off-road racing / \n",
"6-Animation / 61-Art / 97-Drawing / 479-Graffiti / 1385-Street art / \n",
"31-Disc jockey / \n",
"0-Games / 257-Counter-Strike / 351-Jumping / \n",
"1-Vehicle / 8-Football / \n",
"256-Tennis / 974-Racket / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 72-iPhone / 342-Windows Media Video / \n",
"20-Trailer / 743-Edward Cullen / \n",
"3-Concert / 16-Performance art / \n",
"1-Vehicle / 50-Aircraft / 129-Aviation / 223-Weather / 502-Sky / 929-Aerobatics / \n",
"15-Cartoon / 243-Sonic the Hedgehog / 251-Sonic the Hedgehog / \n",
"1-Vehicle / 2-Video game / 4-Car / 47-Personal computer / 228-Laptop / 525-Printer / \n",
"141-Microsoft Windows / \n",
"12-Food / 227-Water / 640-Bottle / \n",
"8-Football / \n",
"10-Animal / 48-Pet / 71-Dog / 280-Wildlife / 657-Autumn / \n",
"10-Animal / 18-Outdoor recreation / 42-Fishing / \n",
"5-Dance / 16-Performance art / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 165-League of Legends / 643-Hero / 2004-Riot Games / \n",
"0-Games / 60-Basketball / 168-Basketball moves / \n",
"1-Vehicle / 64-Transport / 286-Bus / 2226-Bus driver / \n",
"1-Vehicle / 92-Radio-controlled model / 149-Model aircraft / 160-Radio-controlled aircraft / 266-Helicopter / 392-Radio-controlled helicopter / \n",
"61-Art / \n",
"0-Games / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / \n",
"49-School / \n",
"1-Vehicle / 25-Toy / 62-Train / 507-Thomas the Tank Engine / 1685-Toy train / \n",
"30-Drummer / 39-Drums / 44-Drums / 146-Snare drum / 1825-Cajón / 3066-Timbales / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / \n",
"117-Boxing / \n",
"5-Dance / 49-School / \n",
"6-Animation / 15-Cartoon / \n",
"1-Vehicle / 10-Animal / 523-Cattle / 2359-Calf / \n",
"6-Animation / 15-Cartoon / 61-Art / 97-Drawing / 148-Painting / 183-Manga / \n",
"12-Food / 26-Cooking / 215-Eating / 295-Chocolate / 582-Candy / 985-Strawberry / \n",
"25-Toy / 61-Art / 417-Play-Doh / 587-Clay / \n",
"3-Concert / 7-Musician / \n",
"5-Dance / 49-School / 95-Talent show / 130-Gymnastics / 191-Cheerleading / \n",
"15-Cartoon / 61-Art / 97-Drawing / \n",
"3-Concert / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 39-Drums / \n",
"51-Strategy video game / 174-Farm / 373-Clash of Clans / \n",
"20-Trailer / \n",
"10-Animal / 80-Horse / \n",
"0-Games / 185-Dragon Ball / \n",
"205-Newscaster / 223-Weather / \n",
"0-Games / 2-Video game / 577-Penguin / \n",
"1958-Monastery / \n",
"345-Loudspeaker / \n",
"51-Strategy video game / 366-Dragon / 373-Clash of Clans / \n",
"47-Personal computer / 131-Computer / 228-Laptop / 513-Computer keyboard / \n",
"28-Fashion / 415-Sewing / 2873-Kimono / \n",
"12-Food / 25-Toy / 126-The Walt Disney Company / 354-Egg / 387-Spider-Man / 611-Kinder Surprise / 713-Egg / \n",
"3-Concert / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"0-Games / 2-Video game / 127-Xbox 360 / 1007-Portal / 1274-Portal 2 / \n",
"0-Games / 599-Dofus / \n",
"25-Toy / 3357-Monica's Gang / \n",
"66-Bollywood / \n",
"8-Football / \n",
"783-Zee TV / \n",
"0-Games / 2-Video game / \n",
"1-Vehicle / 67-Cycling / 69-Bicycle / 213-Forest / 220-Mountain bike / 284-Mountain biking / \n",
"6-Animation / 10-Animal / 15-Cartoon / 126-The Walt Disney Company / \n",
"7-Musician / 14-Guitar / 100-Electric guitar / 1747-Erhu / \n",
"59-Winter sport / 112-Ice skating / 142-Hockey / 623-Coach / \n",
"79-American football / \n",
"6-Animation / 4708-Visual novel / \n",
"0-Games / 6-Animation / 99-Christmas / 679-Mickey Mouse / \n",
"0-Games / 59-Winter sport / 77-Arena / 112-Ice skating / 142-Hockey / 312-Ice rink / \n",
"155-Camera / 278-GoPro / \n",
"15-Cartoon / 538-Illustration / 4122-Marceline the Vampire Queen / \n",
"5-Dance / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"1-Vehicle / 27-Motorcycle / \n",
"137-Tree / 158-Tire / 315-Knife / 1468-Piston / \n",
"8-Football / 21-Stadium / 77-Arena / \n",
"0-Games / 535-Slot machine / 2301-Dead or Alive / \n",
"8-Football / \n",
"3770-Toothpaste / \n",
"551-British Broadcasting Corporation / \n",
"3-Concert / 5-Dance / \n",
"6-Animation / 61-Art / 97-Drawing / 183-Manga / 528-Pencil / \n",
"3-Concert / \n",
"12-Food / 389-Restaurant / \n",
"23-Mobile phone / 29-Smartphone / 37-Gadget / 101-Telephone / \n",
"0-Games / 2-Video game / 6-Animation / 10-Animal / 132-Bird / 847-Angry Birds / 3319-Angry Birds / \n",
"23-Mobile phone / 29-Smartphone / 72-iPhone / \n",
"13-Musical ensemble / 2570-Constellation / \n",
"298-Television advertisement / \n",
"47-Personal computer / 145-Tablet computer / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 47-Personal computer / 201-Grand Theft Auto: San Andreas / \n",
"0-Games / 2-Video game / 55-Video game console / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 57-Race track / 378-Drag racing / \n",
"7-Musician / 14-Guitar / 24-String instrument / 639-Fiddle / 670-Banjo / \n",
"47-Personal computer / 228-Laptop / \n",
"1-Vehicle / 50-Aircraft / 82-Airplane / 92-Radio-controlled model / 129-Aviation / 149-Model aircraft / 157-Jet aircraft / 160-Radio-controlled aircraft / 388-Fighter aircraft / 2510-McDonnell Douglas F/A-18 Hornet / \n",
"275-Diving / 287-Human swimming / \n",
"12-Food / 274-Meat / \n",
"1311-Amazon.com; Inc. / \n",
"827-Pump / \n",
"3-Concert / 5-Dance / 16-Performance art / 133-Nightclub / \n",
"0-Games / 19-PC game / 35-Minecraft / \n",
"6-Animation / 12-Food / 536-Glass / \n",
"45-Cosmetics / 56-Hair / 78-Wedding / 218-Eyelash / \n",
"467-Chipmunk / \n",
"204-Gym / 532-Barbell / 1584-Kettlebell / 4441-Functional training / \n",
"1-Vehicle / 18-Outdoor recreation / 144-Amusement park / 321-Roller coaster / 334-Amusement ride / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / 352-Need for Speed / 875-Need for Speed: Most Wanted / 1577-Need for Speed: Most Wanted / \n",
"10-Animal / 48-Pet / 132-Bird / 598-Parrot / \n",
"0-Games / 51-Strategy video game / 373-Clash of Clans / \n",
"6-Animation / 15-Cartoon / 61-Art / 97-Drawing / 183-Manga / 185-Dragon Ball / 244-Goku / \n",
"1264-Head / \n",
"0-Games / 1290-Dragon Nest / \n",
"36-Piano / \n",
"10-Animal / 1108-Mushroom / \n",
"5-Dance / 16-Performance art / \n",
"0-Games / 21-Stadium / 54-Highlight film / 81-Athlete / 562-Pitcher / 946-Baseball park / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / \n",
"1035-Bank / \n",
"12-Food / 56-Hair / \n",
"0-Games / 2-Video game / 33-Weapon / 43-Call of Duty / 216-Call of Duty: Modern Warfare 3 / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / \n",
"9-Music video / 15-Cartoon / \n",
"60-Basketball / 2547-Staples Center / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"1-Vehicle / 22-Nature / 42-Fishing / 87-Boat / 92-Radio-controlled model / 1490-Radio-controlled boat / 3108-Fishing vessel / \n",
"2573-Domestic pigeon / \n",
"1424-Coupon / \n",
"5-Dance / 9-Music video / \n",
"0-Games / 2-Video game / 35-Minecraft / 174-Farm / \n",
"6-Animation / 10-Animal / \n",
"0-Games / 1-Vehicle / 4-Car / 17-Racing / 2400-Asphalt / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"0-Games / 2-Video game / 22-Nature / 734-DayZ / 736-ARMA / 1034-ARMA 2 / 1314-Garage / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 16-Performance art / 38-Orchestra / \n",
"444-Family / \n",
"31-Disc jockey / \n",
"581-PlayStation / 973-Nintendo 64 / \n",
"5-Dance / 7-Musician / 13-Musical ensemble / 203-Marching band / 2134-Teddy bear / \n",
"12-Food / 88-Machine / \n",
"1-Vehicle / 50-Aircraft / 139-Airport / 157-Jet aircraft / 451-Cockpit / 1705-Boeing 767 / \n",
"45-Cosmetics / 282-Skin / 385-Face / \n",
"915-Town / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"5-Dance / 16-Performance art / \n",
"3-Concert / 133-Nightclub / \n",
"9-Music video / \n",
"33-Weapon / 273-News program / \n",
"61-Art / 457-Knitting / 546-Stitch / 653-Thread / \n",
"7-Musician / 9-Music video / 13-Musical ensemble / 14-Guitar / \n",
"115-Ballet / \n",
"31-Disc jockey / \n",
"1-Vehicle / 27-Motorcycle / 271-Tank / \n",
"6-Animation / 10-Animal / 323-Toddler / \n",
"3-Concert / 16-Performance art / \n",
"18-Outdoor recreation / 163-Running / 674-Marathon / \n",
"0-Games / 2-Video game / 638-Gears of War / \n",
"6-Animation / 15-Cartoon / 2047-Popeye / \n",
"12-Food / 18-Outdoor recreation / 58-Cuisine / 239-Roasting / 770-Banana / \n",
"141-Microsoft Windows / 512-Printing / 525-Printer / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 77-Arena / 81-Athlete / \n",
"0-Games / 60-Basketball / 168-Basketball moves / 255-Slam dunk / \n",
"286-Bus / 387-Spider-Man / \n",
"248-Hotel / \n",
"3-Concert / 68-Lighting / \n",
"427-Trumpet / 499-Cloud / 502-Sky / \n",
"293-Slide show / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 24-String instrument / 39-Drums / 689-Harmonica / \n",
"14-Guitar / 63-Acoustic guitar / \n",
"1-Vehicle / 4-Car / 40-Road / 167-Four-wheel drive / 198-Off-road vehicle / 286-Bus / 299-Volkswagen Passenger Cars / 327-Mud / \n",
"5-Dance / 95-Talent show / 191-Cheerleading / \n",
"20-Trailer / 886-James Bond / \n",
"0-Games / 81-Athlete / 562-Pitcher / 946-Baseball park / 2150-Baseball bat / \n",
"0-Games / 8-Football / 54-Highlight film / \n",
"12-Food / 26-Cooking / 32-Recipe / 58-Cuisine / 424-Chicken / 557-Salad / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 27-Motorcycle / 40-Road / \n",
"14-Guitar / 100-Electric guitar / \n",
"13-Musical ensemble / \n",
"0-Games / 2-Video game / 55-Video game console / 127-Xbox 360 / 181-Xbox / 1027-Terraria / \n",
"694-Tai chi / \n",
"5-Dance / 16-Performance art / 708-Mermaid / \n",
"0-Games / 19-PC game / 51-Strategy video game / 165-League of Legends / \n",
"3-Concert / 7-Musician / 545-Quartet (ensemble) / \n",
"309-Computer hardware / \n",
"61-Art / \n",
"23-Mobile phone / 29-Smartphone / 178-Samsung Galaxy / 1727-Samsung Galaxy Note 3 / \n",
"0-Games / 8-Football / 54-Highlight film / 76-Ball / 358-Rugby football / \n",
"22-Nature / 86-Plant / 136-Gardening / 279-Garden / \n",
"3-Concert / 21-Stadium / \n",
"0-Games / 2-Video game / 229-Halo / 518-Mario Kart / 644-Halo: Reach / 3225-Princess Daisy / \n",
"10-Animal / 280-Wildlife / \n",
"0-Games / 49-School / 54-Highlight film / 60-Basketball / 168-Basketball moves / \n",
"0-Games / 6-Animation / \n",
"8-Football / \n",
"842-Harp / \n",
"215-Eating / \n",
"0-Games / 307-Hand / 311-Street Fighter / \n",
"141-Microsoft Windows / \n",
"1533-Lead guitar / \n",
"7-Musician / 14-Guitar / 24-String instrument / 63-Acoustic guitar / \n",
"45-Cosmetics / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 40-Road / 41-Sports car / 57-Race track / \n",
"0-Games / 60-Basketball / 77-Arena / 168-Basketball moves / \n",
"1738-Elmo / 3668-Ernie / \n",
"248-Hotel / \n",
"10-Animal / 48-Pet / 71-Dog / 182-Puppy / 1839-Rottweiler / \n",
"28-Fashion / 1038-Hijab / 1096-Scarf / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 85-Combat / 96-Soldier / 192-Battlefield / 383-Battlefield 4 / \n",
"1-Vehicle / 4-Car / 74-Truck / 249-Heavy equipment / 817-Bulldozer / \n",
"9-Music video / \n",
"6-Animation / 183-Manga / 541-Black-and-white / \n",
"0-Games / 10-Animal / \n",
"9-Music video / \n",
"8-Football / 358-Rugby football / \n",
"45-Cosmetics / 415-Sewing / 676-Handbag / 1941-Pattern / \n",
"3-Concert / 5-Dance / \n",
"0-Games / 6-Animation / 10-Animal / 15-Cartoon / \n",
"0-Games / 2-Video game / 19-PC game / 928-Borderlands 2 / \n",
"14-Guitar / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 84-Snow / 120-Winter / 135-Drifting / 349-Chevrolet / 1145-Winter storm / \n",
"10-Animal / 91-Fish / 215-Eating / 258-Aquarium / \n",
"6-Animation / 701-Beyblade / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 1661-Volkswagen Gol / \n",
"9-Music video / \n",
"3-Concert / 16-Performance art / \n",
"1-Vehicle / 74-Truck / 158-Tire / 3431-Volvo FH / \n",
"9-Music video / \n",
"0-Games / 411-MapleStory / 1178-Quest / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 172-Motocross / \n",
"5-Dance / \n",
"0-Games / 1-Vehicle / 2-Video game / 11-Motorsport / 199-Wii / 443-Wii U / 518-Mario Kart / 1831-Mario Kart 8 / \n",
"0-Games / 2-Video game / \n",
"0-Games / 8-Football / 21-Stadium / 54-Highlight film / 65-Kick / 73-Ball / 76-Ball / \n",
"1-Vehicle / \n",
"5-Dance / 6-Animation / \n",
"1-Vehicle / 4-Car / 25-Toy / 962-McDonald's / 1169-Hot Wheels / 1418-Model car / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"5-Dance / 16-Performance art / 95-Talent show / 355-Comedian / \n",
"9-Music video / \n",
"49-School / \n",
"0-Games / 1-Vehicle / 2-Video game / 4-Car / 34-Action-adventure game / 201-Grand Theft Auto: San Andreas / 858-Carl Johnson / \n",
"1-Vehicle / 4-Car / 27-Motorcycle / 94-Dashcam / 4048-Yamaha MT-09 / \n",
"9-Music video / 133-Nightclub / \n",
"5-Dance / 14-Guitar / 95-Talent show / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 27-Motorcycle / 105-Motorcycling / 172-Motocross / 197-Trail / 491-Enduro / 608-Wheelie / \n",
"298-Television advertisement / 337-Advertising / 1035-Bank / \n",
"0-Games / 51-Strategy video game / 285-Sasuke Uchiha / 382-Dota 2 / 549-Defense of the Ancients / \n",
"37-Gadget / 47-Personal computer / 145-Tablet computer / \n",
"395-Camping / \n",
"0-Games / 2-Video game / 317-PlayStation / 413-PlayStation 2 / 426-Resident Evil / 581-PlayStation / \n",
"31-Disc jockey / \n",
"20-Trailer / \n",
"3-Concert / \n",
"0-Games / 8-Football / 54-Highlight film / 81-Athlete / \n",
"126-The Walt Disney Company / 144-Amusement park / \n",
"242-Paper / 370-Origami / 577-Penguin / \n",
"3-Concert / 153-Dress / 356-Star / \n",
"93-Comedy / \n",
"0-Games / 116-Pokémon / 186-Pokémon / 1204-Pokémon Ruby and Sapphire / 1387-Pokémon Battle Revolution / 1665-Pokémon Emerald / \n",
"9-Music video / \n",
"3-Concert / \n",
"66-Bollywood / \n",
"22-Nature / 164-Beach / 233-Ocean / 423-Coast / \n",
"49-School / 661-Stretching / 898-Pilates / 1055-Back / \n",
"45-Cosmetics / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 57-Race track / 787-Monster truck / 3920-Maximum Destruction / \n",
"3428-Costume jewelry / \n",
"20-Trailer / \n",
"20-Trailer / 263-Light / 419-Earth / \n",
"1016-Glove / 3757-Baseball glove / 4241-Wilson Sporting Goods / \n",
"5-Dance / \n",
"25-Toy / 140-LEGO / 359-Harry Potter (Literary Series) / 884-Lego minifigure / \n",
"45-Cosmetics / 193-Eye liner / 195-Mascara / 218-Eyelash / 235-Lipstick / 291-Rouge / 436-Concealer / 1012-Lip gloss / \n",
"0-Games / 8-Football / 21-Stadium / 76-Ball / \n",
"3-Concert / 5-Dance / 68-Lighting / \n",
"0-Games / 2-Video game / 51-Strategy video game / 792-DarkOrbit / \n",
"9-Music video / \n",
"3-Concert / 16-Performance art / \n",
"93-Comedy / \n",
"8-Football / \n",
"0-Games / 2-Video game / 90-Sports game / 480-FIFA 13 / \n",
"3-Concert / 7-Musician / 16-Performance art / \n",
"3-Concert / \n",
"3377-Wii Remote / \n",
"5-Dance / 16-Performance art / \n",
"0-Games / 8-Football / 21-Stadium / 65-Kick / 73-Ball / 76-Ball / 81-Athlete / \n",
"5-Dance / 16-Performance art / 95-Talent show / 130-Gymnastics / 351-Jumping / 1157-Rope / 3830-Double Dutch / \n",
"412-Monster / \n",
"606-String / 1326-Recorder / \n",
"61-Art / 1488-Mosaic / 3266-Ancient Rome / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 17-Racing / 41-Sports car / 57-Race track / 135-Drifting / 169-Rallying / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / 46-Choir / 1747-Erhu / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 46-Choir / 209-University / \n",
"31-Disc jockey / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 36-Piano / 39-Drums / 53-Keyboard / 104-Pianist / 107-Musical keyboard / \n",
"9-Music video / 322-Mixtape / 2461-DatPiff / \n",
"20-Trailer / \n",
"8-Football / \n",
"10-Animal / 80-Horse / 247-Stallion / 290-Pony / 438-Mare / \n",
"0-Games / 2-Video game / 55-Video game console / 396-One Piece / \n",
"1-Vehicle / 87-Boat / \n",
"66-Bollywood / 93-Comedy / \n",
"25-Toy / 200-Doll / 290-Pony / 343-My Little Pony / 417-Play-Doh / 439-Barbie / 524-Littlest Pet Shop / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"5-Dance / 16-Performance art / 59-Winter sport / 112-Ice skating / 115-Ballet / 316-Figure skating / \n",
"28-Fashion / 121-Photography / 153-Dress / 390-Model / \n",
"22-Nature / 67-Cycling / 69-Bicycle / 197-Trail / 220-Mountain bike / 284-Mountain biking / \n",
"47-Personal computer / 131-Computer / 414-Computer monitor / 429-Website / \n",
"5-Dance / 16-Performance art / 115-Ballet / \n",
"10-Animal / \n",
"0-Games / 2-Video game / 19-PC game / 686-Diablo III / 872-Diablo / \n",
"0-Games / 8-Football / 54-Highlight film / 81-Athlete / \n",
"12-Food / \n",
"5-Dance / 78-Wedding / 190-Bride / 421-Gown / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 41-Sports car / 70-Driving / 344-Coupé / 531-Hatchback / 683-Renault / 2455-Renault Mégane / \n",
"72-iPhone / \n",
"3-Concert / 68-Lighting / \n",
"2608-Strap / \n",
"0-Games / 2-Video game / 35-Minecraft / \n",
"5-Dance / 44-Drums / \n",
"1-Vehicle / 11-Motorsport / 17-Racing / 57-Race track / 59-Winter sport / 84-Snow / 120-Winter / 262-Ice / 610-Snowmobile / \n",
"0-Games / 19-PC game / 211-RuneScape / \n",
"18-Outdoor recreation / 96-Soldier / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"415-Sewing / 653-Thread / 997-Sewing machine / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 38-Orchestra / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 14-Guitar / 30-Drummer / 39-Drums / 44-Drums / 100-Electric guitar / \n",
"12-Food / 2129-Vending machine / \n",
"1-Vehicle / 88-Machine / 249-Heavy equipment / 512-Printing / 817-Bulldozer / 934-Flooring / \n",
"30-Drummer / 39-Drums / 44-Drums / 128-Cymbal / 146-Snare drum / \n",
"1-Vehicle / 4-Car / 41-Sports car / 1324-Roadster / \n",
"12-Food / 1363-Candle / \n",
"78-Wedding / \n",
"9-Music video / \n",
"10-Animal / 540-Insect / 775-Bee / 784-Honey / 1121-Beehive / 2858-Gelatin dessert / \n",
"3-Concert / 16-Performance art / 30-Drummer / 68-Lighting / \n",
"1459-Magazine / \n",
"0-Games / 2-Video game / 43-Call of Duty / 134-Call of Duty: Black Ops / \n",
"0-Games / 2-Video game / 19-PC game / 33-Weapon / 1394-Alliance of Valiant Arms / \n",
"12-Food / 454-Sauce / 1022-Chili pepper / 3317-Salsa / \n",
"3-Concert / 5-Dance / 16-Performance art / \n",
"0-Games / \n",
"136-Gardening / 329-Swimming pool / \n",
"0-Games / 2-Video game / 34-Action-adventure game / 118-Grand Theft Auto V / \n",
"8-Football / 21-Stadium / \n",
"6-Animation / 15-Cartoon / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 30-Drummer / 39-Drums / \n",
"0-Games / 39-Drums / 724-Rock Band / \n",
"20-Trailer / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 70-Driving / \n",
"0-Games / 54-Highlight film / 60-Basketball / 76-Ball / 168-Basketball moves / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / 221-Skatepark / \n",
"5-Dance / 430-Ballroom dance / \n",
"0-Games / 2-Video game / 19-PC game / 51-Strategy video game / 165-League of Legends / \n",
"3-Concert / 7-Musician / 13-Musical ensemble / 842-Harp / \n",
"3889-Periodic Table / \n",
"5-Dance / 7-Musician / 78-Wedding / \n",
"0-Games / 6-Animation / 25-Toy / 140-LEGO / \n",
"9-Music video / 14-Guitar / \n",
"310-The Sims / 543-The Sims 2 / \n",
"0-Games / 2-Video game / 20-Trailer / 23-Mobile phone / 29-Smartphone / 72-iPhone / 189-iPad / \n",
"99-Christmas / \n",
"46-Choir / 49-School / \n",
"20-Trailer / \n",
"31-Disc jockey / \n",
"31-Disc jockey / \n",
"122-Weight training / 204-Gym / 1250-Biceps / \n",
"3-Concert / \n",
"31-Disc jockey / \n",
"1-Vehicle / 4-Car / 11-Motorsport / 169-Rallying / \n",
"0-Games / 6-Animation / \n",
"0-Games / 2-Video game / 47-Personal computer / \n",
"93-Comedy / 341-Sketch comedy / \n",
"0-Games / 116-Pokémon / \n",
"5-Dance / \n",
"45-Cosmetics / 56-Hair / 173-Eye shadow / 235-Lipstick / \n",
"89-Comics / \n",
"3535-Boutique / \n",
"36-Piano / 53-Keyboard / 104-Pianist / 107-Musical keyboard / 265-Electronic keyboard / \n",
"18-Outdoor recreation / 22-Nature / 42-Fishing / 147-River / 722-Waterfall / \n",
"6-Animation / 10-Animal / 15-Cartoon / 126-The Walt Disney Company / 571-Bear / 784-Honey / 2089-Winnie-the-Pooh / \n",
"551-British Broadcasting Corporation / \n",
"250-Parade / 253-Carnival / \n",
"5-Dance / 13-Musical ensemble / 16-Performance art / 98-Festival / \n",
"75-Wrestling / \n",
"14-Guitar / 24-String instrument / 100-Electric guitar / \n",
"762-Angel / \n",
"504-Web page / 819-World Wide Web / 3233-Screencast / \n",
"18-Outdoor recreation / 83-Skateboarding / 108-Skateboard / \n",
"1-Vehicle / 4-Car / 70-Driving / 2253-CNET / \n",
"14-Guitar / 100-Electric guitar / \n",
"6-Animation / \n",
"0-Games / 2-Video game / 34-Action-adventure game / \n",
"0-Games / 2-Video game / 19-PC game / 1188-Uncharted / \n",
"125-Television / 504-Web page / 3166-Satellite television / \n",
"36-Piano / 53-Keyboard / \n",
"3-Concert / \n"
]
}
],
"source": [
"i = 0\n",
"for item in validate_labels.iterrows():\n",
" labels = item[1][\"labels\"]\n",
" for l in sorted(map(int, labels.split())):\n",
" print \"%d-%s\" % (l, label[l]), \"/\",\n",
" print \"\"\n",
" i += 1\n",
" if i >= 10000:\n",
" break"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.12"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
================================================
FILE: eda/vertical.tsv
================================================
0 0
1 1
2 0
3 2
4 1
5 2
6 2
7 2
8 3
9 2
10 4
11 3
12 5
13 2
14 2
15 2
16 2
17 3
18 6
19 0
20 2
21 2
22 7
23 8
24 2
25 9
26 5
27 1
28 10
29 8
30 2
31 2
32 5
33 11
34 0
35 0
36 2
37 12
38 2
39 2
40 13
41 1
42 6
43 0
44 2
45 10
46 2
47 12
48 4
49 14
50 1
51 0
52 5
53 12
54 2
55 12
56 10
57 0
58 5
59 3
60 3
61 2
62 13
63 2
64 13
65 3
66 2
67 3
68 15
69 1
70 1
71 4
72 12
73 3
74 1
75 3
76 3
77 2
78 6
79 3
80 4
81 3
82 1
83 3
84 16
85 3
86 7
87 1
88 13
89 2
90 0
91 13
92 6
93 2
94 12
95 2
96 11
97 2
98 2
99 6
100 2
101 8
102 2
103 13
104 2
105 1
106 10
107 2
108 3
109 2
110 2
111 12
112 3
113 15
114 1
115 2
116 2
117 3
118 0
119 13
120 16
121 2
122 10
123 2
124 13
125 2
126 2
127 12
128 2
129 13
130 3
131 12
132 4
133 2
134 0
135 3
136 15
137 13
138 0
139 17
140 9
141 12
142 3
143 13
144 17
145 12
146 2
147 18
148 2
149 13
150 2
151 13
152 13
153 9
154 1
155 12
156 13
157 13
158 1
159 0
160 6
161 13
162 19
163 3
164 17
165 0
166 4
167 1
168 3
169 3
170 15
171 0
172 3
173 10
174 13
175 13
176 12
177 3
178 12
179 15
180 0
181 12
182 4
183 2
184 5
185 2
186 0
187 13
188 17
189 12
190 6
191 3
192 0
193 10
194 0
195 10
196 7
197 13
198 1
199 12
200 9
201 0
202 6
203 2
204 10
205 16
206 9
207 1
208 13
209 14
210 15
211 0
212 18
213 13
214 7
215 5
216 0
217 6
218 10
219 2
220 1
221 3
222 3
223 16
224 13
225 17
226 5
227 13
228 12
229 0
230 2
231 1
232 5
233 7
234 1
235 10
236 5
237 0
238 10
239 5
240 0
241 0
242 13
243 0
244 2
245 1
246 15
247 4
248 17
249 13
250 2
251 2
252 13
253 6
254 13
255 3
256 3
257 0
258 4
259 7
260 20
261 13
262 7
263 7
264 2
265 2
266 1
267 7
268 6
269 2
270 1
271 11
272 12
273 2
274 5
275 6
276 21
277 1
278 12
279 15
280 4
281 3
282 7
283 7
284 1
285 2
286 18
287 3
288 7
289 6
290 4
291 10
292 12
293 14
294 2
295 5
296 2
297 0
298 2
299 1
300 10
301 5
302 0
303 2
304 10
305 3
306 14
307 7
308 1
309 12
310 0
311 0
312 3
313 22
314 12
315 6
316 3
317 12
318 2
319 7
320 0
321 17
322 2
323 20
324 7
325 18
326 0
327 7
328 0
329 15
330 23
331 12
332 1
333 9
334 17
335 12
336 1
337 13
338 10
339 2
340 10
341 2
342 12
343 9
344 1
345 12
346 4
347 3
348 5
349 1
350 2
351 3
352 0
353 13
354 5
355 2
356 7
357 2
358 3
359 22
360 1
361 2
362 12
363 7
364 2
365 2
366 20
367 5
368 9
369 17
370 6
371 11
372 0
373 0
374 0
375 1
376 2
377 1
378 3
379 0
380 0
381 6
382 0
383 0
384 0
385 7
386 12
387 2
388 11
389 5
390 10
391 5
392 6
393 13
394 18
395 6
396 2
397 2
398 12
399 3
400 13
401 2
402 9
403 1
404 13
405 10
406 1
407 6
408 2
409 0
410 14
411 0
412 18
413 12
414 12
415 6
416 21
417 20
418 2
419 7
420 0
421 9
422 0
423 18
424 13
425 18
426 0
427 2
428 6
429 8
430 2
431 0
432 0
433 13
434 2
435 9
436 10
437 5
438 4
439 9
440 4
441 0
442 5
443 12
444 20
445 2
446 2
447 13
448 7
449 1
450 2
451 13
452 9
453 0
454 5
455 2
456 3
457 6
458 7
459 13
460 0
461 0
462 15
463 3
464 0
465 2
466 9
467 4
468 7
469 9
470 0
471 13
472 13
473 1
474 0
475 5
476 13
477 2
478 6
479 2
480 0
481 1
482 13
483 9
484 12
485 5
486 0
487 12
488 15
489 0
490 16
491 3
492 13
493 1
494 0
495 10
496 0
497 12
498 1
499 7
500 5
501 3
502 7
503 9
504 8
505 0
506 0
507 2
508 12
509 1
510 6
511 0
512 13
513 12
514 0
515 1
516 12
517 1
518 0
519 1
520 1
521 13
522 1
523 13
524 9
525 12
526 1
527 12
528 2
529 10
530 0
531 1
532 10
533 13
534 5
535 0
536 13
537 17
538 2
539 15
540 7
541 2
542 5
543 0
544 0
545 2
546 6
547 2
548 3
549 0
550 7
551 16
552 1
553 8
554 12
555 3
556 3
557 5
558 6
559 18
560 5
561 0
562 3
563 0
564 0
565 3
566 12
567 12
568 0
569 1
570 3
571 4
572 5
573 1
574 13
575 1
576 17
577 4
578 5
579 1
580 15
581 12
582 5
583 10
584 0
585 15
586 12
587 7
588 4
589 5
590 9
591 2
592 2
593 7
594 2
595 3
596 0
597 5
598 4
599 0
600 5
601 4
602 12
603 0
604 1
605 1
606 2
607 13
608 3
609 10
610 1
611 5
612 5
613 2
614 6
615 0
616 22
617 13
618 1
619 13
620 5
621 12
622 2
623 3
624 22
625 13
626 0
627 15
628 2
629 12
630 2
631 7
632 3
633 2
634 2
635 0
636 4
637 0
638 12
639 2
640 5
641 9
642 7
643 18
644 0
645 1
646 2
647 1
648 2
649 15
650 1
651 3
652 15
653 13
654 7
655 2
656 0
657 16
658 0
659 1
660 13
661 10
662 5
663 17
664 0
665 0
666 13
667 3
668 1
669 3
670 2
671 5
672 6
673 15
674 3
675 12
676 9
677 12
678 2
679 2
680 1
681 13
682 12
683 1
684 5
685 12
686 0
687 0
688 5
689 2
690 0
691 13
692 0
693 3
694 3
695 9
696 9
697 0
698 13
699 3
700 12
701 2
702 0
703 3
704 0
705 12
706 4
707 15
708 18
709 13
710 3
711 12
712 7
713 7
714 2
715 0
716 7
717 2
718 13
719 17
720 5
721 12
722 7
723 4
724 0
725 20
726 2
727 5
728 3
729 6
730 0
731 23
732 0
733 10
734 0
735 2
736 0
737 2
738 2
739 0
740 0
741 13
742 15
743 2
744 15
745 22
746 2
747 13
748 2
749 2
750 2
751 7
752 5
753 1
754 2
755 2
756 0
757 9
758 11
759 5
760 12
761 1
762 20
763 4
764 0
765 3
766 11
767 3
768 13
769 5
770 5
771 5
772 3
773 12
774 7
775 7
776 14
777 0
778 6
779 0
780 10
781 0
782 12
783 2
784 5
785 9
786 5
787 1
788 2
789 3
790 1
791 0
792 0
793 9
794 13
795 5
796 11
797 0
798 10
799 0
800 18
801 2
802 8
803 2
804 9
805 15
806 1
807 13
808 5
809 2
810 18
811 12
812 13
813 3
814 7
815 6
816 2
817 13
818 15
819 8
820 2
821 13
822 13
823 10
824 4
825 7
826 0
827 13
828 6
829 6
830 15
831 0
832 12
833 3
834 12
835 17
836 0
837 0
838 0
839 0
840 18
841 5
842 2
843 2
844 7
845 14
846 4
847 0
848 0
849 0
850 0
851 0
852 9
853 1
854 1
855 2
856 13
857 10
858 0
859 6
860 20
861 2
862 5
863 10
864 1
865 4
866 12
867 5
868 13
869 0
870 20
871 10
872 0
873 13
874 1
875 0
876 2
877 0
878 2
879 13
880 9
881 4
882 13
883 12
884 9
885 2
886 2
887 0
888 15
889 2
890 2
891 1
892 10
893 5
894 9
895 5
896 6
897 2
898 10
899 8
900 12
901 1
902 6
903 0
904 5
905 12
906 13
907 9
908 12
909 10
910 12
911 21
912 13
913 11
914 6
915 17
916 10
917 6
918 1
919 10
920 15
921 0
922 0
923 13
924 6
925 7
926 6
927 3
928 0
929 13
930 3
931 13
932 8
933 1
934 15
935 3
936 2
937 3
938 13
939 21
940 2
941 0
942 0
943 15
944 5
945 1
946 3
947 1
948 0
949 6
950 12
951 2
952 1
953 0
954 10
955 11
956 19
957 12
958 7
959 12
960 3
961 12
962 5
963 0
964 6
965 2
966 5
967 22
968 5
969 1
970 4
971 18
972 13
973 12
974 3
975 7
976 13
977 0
978 2
979 2
980 2
981 2
982 1
983 1
984 13
985 5
986 9
987 13
988 17
989 0
990 2
991 0
992 13
993 3
994 0
995 2
996 11
997 13
998 7
999 4
1000 0
1001 9
1002 2
1003 2
1004 18
1005 0
1006 2
1007 0
1008 6
1009 12
1010 9
1011 13
1012 10
1013 3
1014 2
1015 12
1016 9
1017 9
1018 5
1019 20
1020 1
1021 0
1022 5
1023 7
1024 5
1025 6
1026 23
1027 0
1028 0
1029 0
1030 9
1031 12
1032 8
1033 13
1034 0
1035 23
1036 1
1037 7
1038 20
1039 4
1040 3
1041 1
1042 12
1043 12
1044 5
1045 12
1046 12
1047 6
1048 0
1049 0
1050 0
1051 4
1052 9
1053 6
1054 15
1055 7
1056 1
1057 6
1058 7
1059 2
1060 19
1061 6
1062 2
1063 12
1064 10
1065 2
1066 3
1067 15
1068 0
1069 6
1070 1
1071 0
1072 0
1073 13
1074 1
1075 4
1076 6
1077 2
1078 2
1079 9
1080 0
1081 13
1082 6
1083 0
1084 4
1085 15
1086 12
1087 0
1088 2
1089 12
1090 6
1091 0
1092 12
1093 13
1094 9
1095 20
1096 9
1097 12
1098 6
1099 0
1100 9
1101 3
1102 5
1103 2
1104 2
1105 3
1106 18
1107 11
1108 5
1109 0
1110 7
1111 3
1112 0
1113 13
1114 5
1115 5
1116 13
1117 12
1118 12
1119 12
1120 0
1121 7
1122 1
1123 13
1124 1
1125 0
1126 7
1127 2
1128 10
1129 13
1130 11
1131 0
1132 1
1133 0
1134 17
1135 4
1136 13
1137 0
1138 0
1139 2
1140 0
1141 0
1142 15
1143 2
1144 0
1145 7
1146 3
1147 4
1148 1
1149 2
1150 12
1151 20
1152 13
1153 2
1154 2
1155 12
1156 2
1157 13
1158 13
1159 12
1160 4
1161 0
1162 19
1163 15
1164 2
1165 2
1166 0
1167 0
1168 17
1169 6
1170 12
1171 12
1172 12
1173 2
1174 5
1175 10
1176 5
1177 7
1178 0
1179 13
1180 2
1181 7
1182 3
1183 2
1184 18
1185 6
1186 5
1187 0
1188 0
1189 5
1190 3
1191 20
1192 2
1193 5
1194 1
1195 6
1196 0
1197 4
1198 7
1199 12
1200 1
1201 1
1202 2
1203 6
1204 0
1205 1
1206 12
1207 12
1208 5
1209 5
1210 3
1211 0
1212 9
1213 0
1214 21
1215 4
1216 2
1217 3
1218 5
1219 2
1220 12
1221 1
1222 15
1223 5
1224 5
1225 8
1226 13
1227 5
1228 0
1229 7
1230 5
1231 2
1232 0
1233 1
1234 4
1235 16
1236 6
1237 20
1238 0
1239 3
1240 2
1241 0
1242 1
1243 7
1244 2
1245 11
1246 14
1247 3
1248 0
1249 6
1250 7
1251 0
1252 4
1253 2
1254 17
1255 6
1256 5
1257 13
1258 4
1259 2
1260 12
1261 0
1262 0
1263 1
1264 7
1265 2
1266 2
1267 0
1268 2
1269 10
1270 6
1271 10
1272 13
1273 12
1274 0
1275 0
1276 0
1277 6
1278 4
1279 6
1280 6
1281 15
1282 1
1283 5
1284 5
1285 13
1286 1
1287 12
1288 0
1289 2
1290 0
1291 0
1292 10
1293 0
1294 13
1295 5
1296 7
1297 1
1298 2
1299 0
1300 9
1301 1
1302 9
1303 2
1304 2
1305 6
1306 6
1307 1
1308 15
1309 6
1310 1
1311 9
1312 0
1313 2
1314 15
1315 3
1316 13
1317 1
1318 15
1319 5
1320 10
1321 1
1322 5
1323 12
1324 1
1325 13
1326 2
1327 2
1328 5
1329 15
1330 3
1331 0
1332 5
1333 2
1334 0
1335 3
1336 1
1337 3
1338 20
1339 0
1340 13
1341 4
1342 2
1343 0
1344 12
1345 2
1346 2
1347 5
1348 12
1349 12
1350 13
1351 3
1352 2
1353 20
1354 4
1355 5
1356 0
1357 6
1358 7
1359 10
1360 2
1361 1
1362 13
1363 18
1364 0
1365 5
1366 8
1367 3
1368 18
1369 9
1370 0
1371 0
1372 1
1373 3
1374 0
1375 2
1376 17
1377 4
1378 0
1379 12
1380 1
1381 5
1382 0
1383 12
1384 15
1385 2
1386 2
1387 0
1388 1
1389 10
1390 12
1391 1
1392 0
1393 2
1394 0
1395 13
1396 0
1397 0
1398 18
1399 3
1400 0
1401 0
1402 17
1403 6
1404 0
1405 14
1406 1
1407 12
1408 5
1409 11
1410 0
1411 2
1412 0
1413 1
1414 13
1415 1
1416 7
1417 3
1418 9
1419 0
1420 7
1421 2
1422 2
1423 9
1424 13
1425 12
1426 18
1427 0
1428 13
1429 1
1430 10
1431 2
1432 0
1433 0
1434 6
1435 0
1436 2
1437 1
1438 5
1439 15
1440 0
1441 16
1442 2
1443 2
1444 4
1445 2
1446 12
1447 0
1448 1
1449 6
1450 6
1451 13
1452 9
1453 6
1454 15
1455 1
1456 15
1457 4
1458 2
1459 22
1460 5
1461 1
1462 0
1463 17
1464 12
1465 2
1466 1
1467 0
1468 1
1469 2
1470 0
1471 5
1472 5
1473 5
1474 3
1475 0
1476 3
1477 3
1478 9
1479 15
1480 13
1481 9
1482 12
1483 11
1484 0
1485 0
1486 1
1487 2
1488 2
1489 15
1490 1
1491 13
1492 0
1493 1
1494 17
1495 3
1496 0
1497 13
1498 15
1499 0
1500 13
1501 11
1502 1
1503 0
1504 12
1505 0
1506 0
1507 2
1508 13
1509 5
1510 7
1511 1
1512 12
1513 0
1514 5
1515 0
1516 9
1517 5
1518 1
1519 1
1520 1
1521 1
1522 0
1523 22
1524 24
1525 9
1526 2
1527 2
1528 23
1529 1
1530 0
1531 7
1532 11
1533 2
1534 10
1535 6
1536 13
1537 1
1538 2
1539 5
1540 1
1541 5
1542 4
1543 0
1544 7
1545 13
1546 7
1547 13
1548 13
1549 2
1550 16
1551 2
1552 14
1553 0
1554 1
1555 2
1556 13
1557 5
1558 9
1559 12
1560 12
1561 6
1562 2
1563 2
1564 0
1565 12
1566 0
1567 13
1568 4
1569 5
1570 0
1571 0
1572 20
1573 11
1574 2
1575 1
1576 5
1577 0
1578 0
1579 18
1580 0
1581 12
1582 0
1583 1
1584 10
1585 0
1586 6
1587 1
1588 0
1589 5
1590 1
1591 4
1592 15
1593 12
1594 20
1595 12
1596 5
1597 5
1598 2
1599 7
1600 0
1601 20
1602 13
1603 4
1604 20
1605 0
1606 7
1607 1
1608 19
1609 3
1610 3
1611 1
1612 0
1613 24
1614 1
1615 0
1616 12
1617 7
1618 0
1619 0
1620 9
1621 20
1622 1
1623 3
1624 1
1625 17
1626 3
1627 9
1628 0
1629 12
1630 24
1631 0
1632 18
1633 11
1634 1
1635 1
1636 6
1637 12
1638 0
1639 4
1640 18
1641 5
1642 9
1643 11
1644 24
1645 4
1646 2
1647 24
1648 24
1649 5
1650 12
1651 1
1652 5
1653 0
1654 2
1655 0
1656 12
1657 12
1658 12
1659 5
1660 0
1661 1
1662 3
1663 7
1664 0
1665 0
1666 2
1667 6
1668 0
1669 5
1670 6
1671 13
1672 12
1673 17
1674 2
1675 2
1676 0
1677 5
1678 18
1679 0
1680 0
1681 0
1682 2
1683 0
1684 20
1685 9
1686 7
1687 0
1688 1
1689 0
1690 0
1691 0
1692 16
1693 5
1694 0
1695 0
1696 2
1697 20
1698 0
1699 1
1700 13
1701 5
1702 12
1703 0
1704 2
1705 13
1706 0
1707 15
1708 0
1709 1
1710 2
1711 3
1712 9
1713 0
1714 18
1715 11
1716 4
1717 0
1718 0
1719 0
1720 5
1721 0
1722 2
1723 4
1724 2
1725 5
1726 20
1727 12
1728 5
1729 5
1730 0
1731 2
1732 18
1733 7
1734 2
1735 0
1736 0
1737 0
1738 2
1739 15
1740 11
1741 5
1742 1
1743 0
1744 2
1745 0
1746 13
1747 24
1748 5
1749 0
1750 4
1751 12
1752 1
1753 15
1754 0
1755 24
1756 2
1757 2
1758 0
1759 12
1760 0
1761 1
1762 0
1763 1
1764 3
1765 5
1766 0
1767 4
1768 15
1769 5
1770 0
1771 2
1772 13
1773 12
1774 12
1775 0
1776 8
1777 12
1778 12
1779 13
1780 4
1781 3
1782 5
1783 3
1784 18
1785 0
1786 13
1787 24
1788 3
1789 0
1790 6
1791 2
1792 15
1793 5
1794 15
1795 9
1796 0
1797 13
1798 1
1799 4
1800 9
1801 2
1802 0
1803 24
1804 6
1805 17
1806 1
1807 1
1808 17
1809 2
1810 0
1811 9
1812 10
1813 0
1814 5
1815 1
1816 0
1817 24
1818 18
1819 0
1820 12
1821 13
1822 13
1823 15
1824 12
1825 2
1826 2
1827 10
1828 1
1829 9
1830 3
1831 0
1832 3
1833 1
1834 10
1835 24
1836 6
1837 0
1838 13
1839 4
1840 1
1841 0
1842 9
1843 5
1844 2
1845 4
1846 2
1847 0
1848 7
1849 16
1850 9
1851 5
1852 0
1853 5
1854 24
1855 12
1856 2
1857 0
1858 2
1859 0
1860 0
1861 0
1862 12
1863 13
1864 12
1865 0
1866 2
1867 1
1868 20
1869 13
1870 5
1871 4
1872 12
1873 3
1874 15
1875 1
1876 0
1877 1
1878 0
1879 6
1880 2
1881 2
1882 11
1883 0
1884 2
1885 7
1886 10
1887 1
1888 2
1889 15
1890 0
1891 0
1892 14
1893 0
1894 18
1895 2
1896 1
1897 1
1898 13
1899 22
1900 2
1901 1
1902 0
1903 1
1904 13
1905 4
1906 12
1907 1
1908 0
1909 2
1910 5
1911 0
1912 2
1913 13
1914 6
1915 0
1916 1
1917 6
1918 7
1919 0
1920 13
1921 3
1922 12
1923 0
1924 22
1925 7
1926 12
1927 12
1928 0
1929 0
1930 2
1931 1
1932 1
1933 6
1934 2
1935 0
1936 2
1937 3
1938 0
1939 12
1940 0
1941 10
1942 2
1943 5
1944 17
1945 0
1946 2
1947 4
1948 4
1949 4
1950 2
1951 1
1952 1
1953 24
1954 12
1955 22
1956 1
1957 1
1958 20
1959 12
1960 1
1961 0
1962 12
1963 0
1964 2
1965 1
1966 2
1967 0
1968 0
1969 7
1970 15
1971 2
1972 9
1973 3
1974 1
1975 12
1976 0
1977 2
1978 13
1979 1
1980 5
1981 12
1982 2
1983 24
1984 1
1985 0
1986 2
1987 2
1988 0
1989 12
1990 0
1991 0
1992 1
1993 4
1994 24
1995 15
1996 12
1997 1
1998 12
1999 5
2000 5
2001 22
2002 0
2003 3
2004 0
2005 0
2006 24
2007 13
2008 24
2009 12
2010 12
2011 2
2012 13
2013 2
2014 9
2015 16
2016 1
2017 13
2018 4
2019 19
2020 1
2021 1
2022 7
2023 4
2024 4
2025 13
2026 0
2027 5
2028 0
2029 4
2030 2
2031 13
2032 5
2033 12
2034 12
2035 13
2036 12
2037 13
2038 23
2039 5
2040 5
2041 0
2042 5
2043 0
2044 10
2045 10
2046 5
2047 2
2048 5
2049 11
2050 24
2051 2
2052 1
2053 2
2054 0
2055 21
2056 11
2057 5
2058 6
2059 4
2060 0
2061 1
2062 7
2063 0
2064 13
2065 2
2066 5
2067 15
2068 13
2069 2
2070 1
2071 2
2072 5
2073 2
2074 0
2075 18
2076 2
2077 7
2078 1
2079 13
2080 5
2081 0
2082 2
2083 4
2084 0
2085 24
2086 0
2087 4
2088 0
2089 2
2090 4
2091 13
2092 13
2093 13
2094 4
2095 23
2096 22
2097 4
2098 2
2099 6
2100 1
2101 0
2102 15
2103 0
2104 1
2105 0
2106 0
2107 18
2108 10
2109 1
2110 2
2111 5
2112 2
2113 0
2114 20
2115 13
2116 2
2117 0
2118 7
2119 2
2120 7
2121 0
2122 2
2123 0
2124 13
2125 5
2126 0
2127 0
2128 20
2129 13
2130 1
2131 5
2132 20
2133 3
2134 9
2135 6
2136 13
2137 0
2138 13
2139 0
2140 0
2141 13
2142 12
2143 0
2144 0
2145 2
2146 2
2147 13
2148 24
2149 2
2150 3
2151 17
2152 20
2153 13
2154 4
2155 18
2156 3
2157 4
2158 2
2159 12
2160 1
2161 6
2162 13
2163 12
2164 13
2165 12
2166 12
2167 9
2168 9
2169 2
2170 0
2171 22
2172 2
2173 0
2174 17
2175 13
2176 2
2177 13
2178 5
2179 0
2180 12
2181 0
2182 12
2183 0
2184 10
2185 2
2186 9
2187 5
2188 9
2189 13
2190 15
2191 8
2192 0
2193 12
2194 13
2195 0
2196 13
2197 5
2198 2
2199 17
2200 1
2201 22
2202 11
2203 1
2204 0
2205 12
2206 13
2207 24
2208 1
2209 13
2210 1
2211 4
2212 0
2213 5
2214 1
2215 1
2216 1
2217 12
2218 13
2219 0
2220 15
2221 13
2222 4
2223 1
2224 22
2225 1
2226 17
2227 2
2228 10
2229 12
2230 5
2231 2
2232 1
2233 2
2234 5
2235 10
2236 7
2237 0
2238 0
2239 0
2240 0
2241 2
2242 15
2243 13
2244 3
2245 18
2246 7
2247 1
2248 12
2249 0
2250 12
2251 0
2252 2
2253 2
2254 0
2255 2
2256 13
2257 9
2258 15
2259 8
2260 0
2261 1
2262 12
2263 12
2264 12
2265 24
2266 5
2267 24
2268 1
2269 2
2270 16
2271 4
2272 1
2273 6
2274 12
2275 3
2276 0
2277 0
2278 0
2279 19
2280 2
2281 15
2282 6
2283 4
2284 7
2285 12
2286 1
2287 3
2288 1
2289 0
2290 24
2291 2
2292 1
2293 0
2294 17
2295 12
2296 12
2297 20
2298 3
2299 2
2300 6
2301 0
2302 24
2303 1
2304 1
2305 1
2306 1
2307 0
2308 1
2309 19
2310 1
2311 7
2312 2
2313 5
2314 1
2315 9
2316 2
2317 1
2318 12
2319 9
2320 0
2321 0
2322 3
2323 17
2324 2
2325 1
2326 11
2327 0
2328 12
2329 13
2330 2
2331 7
2332 2
2333 13
2334 0
2335 1
2336 0
2337 11
2338 1
2339 4
2340 0
2341 2
2342 2
2343 7
2344 12
2345 1
2346 0
2347 13
2348 12
2349 6
2350 2
2351 10
2352 0
2353 0
2354 0
2355 17
2356 10
2357 2
2358 3
2359 13
2360 3
2361 12
2362 6
2363 5
2364 2
2365 1
2366 1
2367 2
2368 2
2369 0
2370 1
2371 5
2372 19
2373 7
2374 7
2375 2
2376 24
2377 1
2378 13
2379 0
2380 2
2381 5
2382 2
2383 11
2384 2
2385 15
2386 5
2387 11
2388 4
2389 10
2390 23
2391 17
2392 9
2393 2
2394 0
2395 5
2396 4
2397 5
2398 11
2399 2
2400 0
2401 24
2402 12
2403 2
2404 1
2405 5
2406 5
2407 0
2408 2
2409 1
2410 4
2411 2
2412 2
2413 13
2414 13
2415 6
2416 12
2417 12
2418 5
2419 15
2420 1
2421 16
2422 24
2423 0
2424 9
2425 13
2426 0
2427 1
2428 13
2429 2
2430 2
2431 0
2432 1
2433 0
2434 0
2435 24
2436 18
2437 15
2438 4
2439 1
2440 0
2441 2
2442 0
2443 0
2444 5
2445 13
2446 1
2447 2
2448 9
2449 5
2450 0
2451 12
2452 2
2453 24
2454 13
2455 1
2456 7
2457 7
2458 2
2459 15
2460 0
2461 2
2462 24
2463 20
2464 22
2465 12
2466 0
2467 2
2468 0
2469 10
2470 0
2471 0
2472 13
2473 13
2474 0
2475 15
2476 0
2477 2
2478 10
2479 4
2480 15
2481 13
2482 15
2483 5
2484 12
2485 3
2486 2
2487 12
2488 10
2489 2
2490 15
2491 0
2492 4
2493 2
2494 1
2495 0
2496 5
2497 0
2498 12
2499 6
2500 5
2501 15
2502 0
2503 0
2504 5
2505 3
2506 0
2507 6
2508 8
2509 1
2510 13
2511 1
2512 1
2513 0
2514 24
2515 0
2516 15
2517 1
2518 0
2519 0
2520 2
2521 7
2522 1
2523 0
2524 0
2525 9
2526 0
2527 0
2528 18
2529 2
2530 1
2531 3
2532 2
2533 12
2534 7
2535 0
2536 2
2537 0
2538 20
2539 0
2540 2
2541 0
2542 20
2543 1
2544 0
2545 0
2546 0
2547 2
2548 15
2549 0
2550 0
2551 2
2552 13
2553 4
2554 0
2555 2
2556 6
2557 0
2558 1
2559 7
2560 0
2561 18
2562 5
2563 5
2564 5
2565 3
2566 2
2567 0
2568 2
2569 0
2570 20
2571 9
2572 2
2573 4
2574 3
2575 5
2576 2
2577 5
2578 13
2579 5
2580 12
2581 12
2582 3
2583 17
2584 2
2585 1
2586 0
2587 17
2588 5
2589 12
2590 0
2591 5
2592 12
2593 5
2594 0
2595 2
2596 12
2597 12
2598 4
2599 13
2600 22
2601 7
2602 0
2603 1
2604 13
2605 3
2606 12
2607 5
2608 6
2609 13
2610 20
2611 3
2612 0
2613 24
2614 5
2615 12
2616 1
2617 1
2618 5
2619 1
2620 11
2621 0
2622 0
2623 13
2624 2
2625 2
2626 4
2627 22
2628 24
2629 12
2630 2
2631 4
2632 13
2633 0
2634 12
2635 13
2636 0
2637 5
2638 12
2639 1
2640 7
2641 20
2642 2
2643 7
2644 0
2645 0
2646 2
2647 0
2648 7
2649 20
2650 15
2651 24
2652 1
2653 6
2654 1
2655 2
2656 2
2657 0
2658 0
2659 0
2660 1
2661 2
2662 24
2663 10
2664 0
2665 12
2666 15
2667 0
2668 0
2669 13
2670 6
2671 12
2672 12
2673 4
2674 0
2675 9
2676 5
2677 0
2678 1
2679 2
2680 6
2681 5
2682 24
2683 1
2684 2
2685 18
2686 1
2687 10
2688 0
2689 4
2690 1
2691 0
2692 4
2693 9
2694 5
2695 24
2696 24
2697 4
2698 0
2699 5
2700 0
2701 1
2702 6
2703 15
2704 0
2705 13
2706 5
2707 15
2708 0
2709 12
2710 2
2711 13
2712 0
2713 1
2714 0
2715 0
2716 22
2717 1
2718 12
2719 13
2720 13
2721 15
2722 23
2723 22
2724 13
2725 18
2726 0
2727 12
2728 5
2729 2
2730 0
2731 19
2732 3
2733 0
2734 5
2735 4
2736 1
2737 13
2738 2
2739 0
2740 2
2741 7
2742 9
2743 15
2744 2
2745 13
2746 2
2747 3
2748 9
2749 5
2750 12
2751 12
2752 0
2753 1
2754 12
2755 12
2756 7
2757 5
2758 4
2759 2
2760 1
2761 18
2762 2
2763 1
2764 2
2765 13
2766 2
2767 2
2768 15
2769 13
2770 12
2771 2
2772 0
2773 12
2774 0
2775 5
2776 7
2777 5
2778 1
2779 0
2780 4
2781 9
2782 2
2783 4
2784 7
2785 9
2786 12
2787 1
2788 1
2789 0
2790 17
2791 12
2792 0
2793 24
2794 2
2795 15
2796 0
2797 24
2798 0
2799 0
2800 1
2801 0
2802 12
2803 20
2804 4
2805 13
2806 0
2807 1
2808 12
2809 2
2810 0
2811 24
2812 0
2813 2
2814 9
2815 12
2816 2
2817 0
2818 12
2819 13
2820 0
2821 7
2822 3
2823 8
2824 1
2825 5
2826 1
2827 2
2828 0
2829 18
2830 12
2831 5
2832 2
2833 0
2834 24
2835 2
2836 6
2837 24
2838 13
2839 3
2840 9
2841 0
2842 9
2843 15
2844 24
2845 11
2846 24
2847 9
2848 2
2849 0
2850 17
2851 24
2852 5
2853 13
2854 0
2855 2
2856 2
2857 1
2858 5
2859 4
2860 12
2861 9
2862 1
2863 0
2864 2
2865 1
2866 3
2867 7
2868 1
2869 0
2870 9
2871 0
2872 1
2873 9
2874 13
2875 1
2876 7
2877 0
2878 2
2879 0
2880 0
2881 15
2882 18
2883 12
2884 0
2885 1
2886 13
2887 9
2888 4
2889 0
2890 2
2891 0
2892 1
2893 0
2894 0
2895 13
2896 0
2897 0
2898 12
2899 1
2900 1
2901 0
2902 2
2903 15
2904 13
2905 0
2906 12
2907 3
2908 12
2909 0
2910 12
2911 2
2912 0
2913 15
2914 9
2915 2
2916 0
2917 1
2918 7
2919 0
2920 13
2921 3
2922 2
2923 4
2924 4
2925 1
2926 13
2927 15
2928 5
2929 8
2930 0
2931 18
2932 1
2933 2
2934 0
2935 6
2936 0
2937 23
2938 7
2939 18
2940 2
2941 5
2942 7
2943 5
2944 24
2945 2
2946 0
2947 0
2948 11
2949 1
2950 6
2951 2
2952 2
2953 2
2954 13
2955 5
2956 15
2957 22
2958 12
2959 15
2960 0
2961 6
2962 2
2963 9
2964 5
2965 2
2966 6
2967 2
2968 24
2969 4
2970 0
2971 2
2972 4
2973 3
2974 2
2975 1
2976 2
2977 4
2978 0
2979 9
2980 2
2981 2
2982 5
2983 9
2984 2
2985 24
2986 15
2987 2
2988 0
2989 1
2990 0
2991 1
2992 5
2993 0
2994 12
2995 5
2996 7
2997 24
2998 0
2999 0
3000 12
3001 0
3002 10
3003 5
3004 1
3005 24
3006 1
3007 4
3008 15
3009 15
3010 9
3011 24
3012 20
3013 3
3014 2
3015 13
3016 0
3017 4
3018 4
3019 8
3020 5
3021 5
3022 4
3023 0
3024 2
3025 2
3026 0
3027 5
3028 0
3029 22
3030 2
3031 2
3032 0
3033 0
3034 10
3035 24
3036 2
3037 5
3038 15
3039 20
3040 1
3041 20
3042 20
3043 12
3044 5
3045 12
3046 4
3047 0
3048 0
3049 5
3050 2
3051 2
3052 4
3053 0
3054 5
3055 4
3056 0
3057 24
3058 1
3059 0
3060 4
3061 7
3062 12
3063 20
3064 0
3065 6
3066 24
3067 0
3068 2
3069 0
3070 15
3071 15
3072 12
3073 13
3074 1
3075 0
3076 5
3077 0
3078 0
3079 2
3080 0
3081 0
3082 0
3083 1
3084 6
3085 24
3086 1
3087 5
3088 15
3089 8
3090 2
3091 1
3092 22
3093 0
3094 13
3095 12
3096 24
3097 12
3098 4
3099 0
3100 5
3101 10
3102 2
3103 5
3104 4
3105 7
3106 10
3107 16
3108 1
3109 2
3110 7
3111 0
3112 2
3113 5
3114 11
3115 0
3116 24
3117 6
3118 2
3119 10
3120 0
3121 4
3122 1
3123 7
3124 24
3125 2
3126 5
3127 1
3128 2
3129 10
3130 12
3131 2
3132 1
3133 12
3134 1
3135 0
3136 1
3137 2
3138 15
3139 5
3140 3
3141 0
3142 15
3143 9
3144 22
3145 6
3146 0
3147 0
3148 2
3149 2
3150 5
3151 2
3152 2
3153 2
3154 9
3155 1
3156 15
3157 4
3158 1
3159 0
3160 0
3161 2
3162 13
3163 1
3164 5
3165 20
3166 2
3167 0
3168 0
3169 2
3170 10
3171 12
3172 2
3173 4
3174 3
3175 0
3176 7
3177 7
3178 15
3179 18
3180 2
3181 24
3182 15
3183 24
3184 12
3185 13
3186 8
3187 1
3188 11
3189 21
3190 1
3191 2
3192 2
3193 5
3194 1
3195 12
3196 12
3197 1
3198 8
3199 2
3200 18
3201 2
3202 20
3203 0
3204 5
3205 5
3206 12
3207 5
3208 24
3209 15
3210 24
3211 5
3212 3
3213 24
3214 2
3215 2
3216 24
3217 12
3218 0
3219 1
3220 24
3221 24
3222 2
3223 4
3224 6
3225 0
3226 2
3227 0
3228 5
3229 1
3230 0
3231 0
3232 2
3233 12
3234 2
3235 0
3236 15
3237 1
3238 0
3239 12
3240 9
3241 2
3242 24
3243 3
3244 12
3245 1
3246 12
3247 2
3248 2
3249 7
3250 5
3251 5
3252 0
3253 0
3254 5
3255 0
3256 15
3257 20
3258 12
3259 0
3260 5
3261 2
3262 24
3263 0
3264 3
3265 4
3266 18
3267 1
3268 4
3269 0
3270 9
3271 10
3272 24
3273 4
3274 24
3275 24
3276 2
3277 0
3278 13
3279 1
3280 7
3281 7
3282 2
3283 0
3284 0
3285 12
3286 3
3287 5
3288 5
3289 2
3290 0
3291 3
3292 6
3293 2
3294 5
3295 0
3296 12
3297 7
3298 12
3299 24
3300 15
3301 15
3302 0
3303 0
3304 24
3305 3
3306 2
3307 12
3308 0
3309 1
3310 7
3311 1
3312 9
3313 5
3314 12
3315 2
3316 1
3317 5
3318 6
3319 0
3320 1
3321 0
3322 12
3323 2
3324 1
3325 0
3326 0
3327 13
3328 12
3329 13
3330 24
3331 13
3332 4
3333 18
3334 1
3335 1
3336 24
3337 24
3338 1
3339 1
3340 2
3341 2
3342 22
3343 2
3344 2
3345 3
3346 0
3347 2
3348 24
3349 2
3350 1
3351 12
3352 7
3353 1
3354 2
3355 0
3356 5
3357 2
3358 18
3359 15
3360 0
3361 8
3362 24
3363 12
3364 5
3365 0
3366 0
3367 2
3368 3
3369 2
3370 3
3371 1
3372 1
3373 1
3374 1
3375 17
3376 12
3377 12
3378 0
3379 5
3380 2
3381 2
3382 5
3383 4
3384 5
3385 5
3386 15
3387 24
3388 0
3389 2
3390 17
3391 2
3392 0
3393 5
3394 15
3395 2
3396 24
3397 0
3398 9
3399 1
3400 15
3401 2
3402 5
3403 4
3404 5
3405 12
3406 0
3407 0
3408 12
3409 1
3410 6
3411 2
3412 2
3413 7
3414 1
3415 0
3416 1
3417 2
3418 0
3419 1
3420 14
3421 24
3422 2
3423 0
3424 12
3425 6
3426 0
3427 0
3428 9
3429 2
3430 0
3431 1
3432 24
3433 16
3434 0
3435 2
3436 2
3437 2
3438 24
3439 1
3440 0
3441 8
3442 14
3443 7
3444 4
3445 2
3446 7
3447 0
3448 0
3449 2
3450 0
3451 0
3452 2
3453 2
3454 2
3455 13
3456 0
3457 15
3458 24
3459 5
3460 0
3461 20
3462 0
3463 20
3464 1
3465 2
3466 24
3467 5
3468 0
3469 24
3470 0
3471 3
3472 13
3473 2
3474 0
3475 1
3476 1
3477 5
3478 20
3479 0
3480 2
3481 24
3482 12
3483 2
3484 2
3485 17
3486 1
3487 5
3488 1
3489 2
3490 17
3491 13
3492 22
3493 6
3494 2
3495 20
3496 1
3497 10
3498 19
3499 4
3500 0
3501 20
3502 15
3503 19
3504 1
3505 9
3506 0
3507 2
3508 22
3509 0
3510 0
3511 7
3512 0
3513 0
3514 0
3515 13
3516 1
3517 0
3518 5
3519 0
3520 9
3521 13
3522 2
3523 2
3524 0
3525 2
3526 0
3527 0
3528 0
3529 1
3530 13
3531 13
3532 12
3533 3
3534 1
3535 9
3536 0
3537 12
3538 0
3539 5
3540 18
3541 1
3542 1
3543 2
3544 13
3545 2
3546 0
3547 1
3548 1
3549 1
3550 6
3551 15
3552 3
3553 3
3554 0
3555 0
3556 2
3557 24
3558 0
3559 15
3560 1
3561 2
3562 2
3563 17
3564 1
3565 0
3566 24
3567 12
3568 2
3569 24
3570 4
3571 5
3572 12
3573 1
3574 5
3575 2
3576 4
3577 12
3578 1
3579 2
3580 10
3581 6
3582 3
3583 13
3584 10
3585 7
3586 0
3587 1
3588 2
3589 1
3590 6
3591 0
3592 1
3593 15
3594 24
3595 2
3596 5
3597 0
3598 5
3599 22
3600 22
3601 1
3602 6
3603 13
3604 3
3605 2
3606 24
3607 5
3608 5
3609 13
3610 5
3611 1
3612 0
3613 5
3614 1
3615 20
3616 2
3617 4
3618 0
3619 12
3620 0
3621 18
3622 2
3623 12
3624 0
3625 5
3626 13
3627 0
3628 23
3629 5
3630 1
3631 4
3632 12
3633 13
3634 24
3635 12
3636 1
3637 2
3638 4
3639 13
3640 24
3641 5
3642 6
3643 24
3644 3
3645 24
3646 1
3647 24
3648 12
3649 0
3650 11
3651 13
3652 3
3653 20
3654 12
3655 10
3656 2
3657 1
3658 4
3659 9
3660 4
3661 2
3662 2
3663 2
3664 12
3665 1
3666 0
3667 24
3668 2
3669 4
3670 2
3671 13
3672 1
3673 13
3674 13
3675 0
3676 0
3677 9
3678 15
3679 3
3680 17
3681 1
3682 9
3683 2
3684 1
3685 13
3686 6
3687 17
3688 12
3689 2
3690 12
3691 9
3692 22
3693 6
3694 16
3695 12
3696 13
3697 1
3698 13
3699 4
3700 24
3701 9
3702 24
3703 12
3704 1
3705 16
3706 24
3707 14
3708 0
3709 1
3710 4
3711 10
3712 18
3713 1
3714 7
3715 12
3716 1
3717 0
3718 12
3719 2
3720 12
3721 0
3722 5
3723 0
3724 0
3725 5
3726 0
3727 12
3728 5
3729 24
3730 6
3731 0
3732 8
3733 2
3734 2
3735 5
3736 15
3737 1
3738 3
3739 2
3740 2
3741 2
3742 1
3743 5
3744 2
3745 4
3746 2
3747 0
3748 1
3749 12
3750 24
3751 15
3752 13
3753 12
3754 2
3755 1
3756 15
3757 3
3758 1
3759 0
3760 1
3761 10
3762 1
3763 4
3764 0
3765 13
3766 13
3767 5
3768 0
3769 17
3770 10
3771 1
3772 24
3773 0
3774 0
3775 1
3776 1
3777 13
3778 1
3779 14
3780 13
3781 9
3782 9
3783 9
3784 0
3785 4
3786 0
3787 12
3788 7
3789 15
3790 24
3791 0
3792 2
3793 1
3794 9
3795 2
3796 12
3797 7
3798 4
3799 5
3800 12
3801 0
3802 5
3803 4
3804 2
3805 2
3806 12
3807 0
3808 19
3809 17
3810 2
3811 1
3812 2
3813 2
3814 0
3815 0
3816 2
3817 9
3818 24
3819 2
3820 13
3821 2
3822 7
3823 0
3824 1
3825 5
3826 1
3827 5
3828 1
3829 5
3830 3
3831 9
3832 24
3833 3
3834 1
3835 5
3836 1
3837 3
3838 15
3839 1
3840 0
3841 0
3842 6
3843 0
3844 0
3845 10
3846 0
3847 2
3848 1
3849 0
3850 18
3851 12
3852 4
3853 2
3854 12
3855 0
3856 13
3857 5
3858 1
3859 0
3860 5
3861 10
3862 2
3863 0
3864 6
3865 0
3866 12
3867 8
3868 24
3869 5
3870 3
3871 0
3872 0
3873 1
3874 0
3875 24
3876 5
3877 2
3878 10
3879 19
3880 15
3881 1
3882 24
3883 0
3884 0
3885 10
3886 1
3887 13
3888 6
3889 7
3890 1
3891 5
3892 0
3893 24
3894 0
3895 3
3896 1
3897 0
3898 15
3899 0
3900 12
3901 0
3902 0
3903 12
3904 24
3905 2
3906 0
3907 0
3908 13
3909 1
3910 5
3911 13
3912 3
3913 0
3914 18
3915 1
3916 1
3917 0
3918 12
3919 7
3920 24
3921 24
3922 2
3923 4
3924 12
3925 17
3926 12
3927 0
3928 2
3929 13
3930 0
3931 7
3932 0
3933 2
3934 15
3935 6
3936 24
3937 13
3938 7
3939 1
3940 17
3941 1
3942 7
3943 12
3944 9
3945 1
3946 8
3947 24
3948 6
3949 0
3950 12
3951 7
3952 0
3953 2
3954 2
3955 1
3956 5
3957 1
3958 12
3959 0
3960 24
3961 0
3962 12
3963 12
3964 0
3965 0
3966 0
3967 17
3968 13
3969 0
3970 1
3971 19
3972 2
3973 10
3974 12
3975 22
3976 0
3977 0
3978 12
3979 1
3980 12
3981 24
3982 12
3983 24
3984 12
3985 0
3986 4
3987 3
3988 2
3989 1
3990 1
3991 1
3992 0
3993 20
3994 2
3995 1
3996 2
3997 1
3998 2
3999 0
4000 1
4001 20
4002 2
4003 3
4004 0
4005 1
4006 24
4007 0
4008 1
4009 0
4010 0
4011 18
4012 6
4013 13
4014 1
4015 1
4016 24
4017 2
4018 2
4019 0
4020 1
4021 5
4022 5
4023 0
4024 9
4025 24
4026 1
4027 24
4028 4
4029 2
4030 2
4031 12
4032 0
4033 13
4034 1
4035 13
4036 1
4037 5
4038 9
4039 6
4040 20
4041 1
4042 24
4043 0
4044 24
4045 13
4046 0
4047 0
4048 1
4049 0
4050 13
4051 23
4052 2
4053 5
4054 24
4055 13
4056 12
4057 0
4058 0
4059 8
4060 1
4061 0
4062 0
4063 0
4064 0
4065 5
4066 15
4067 1
4068 0
4069 12
4070 0
4071 1
4072 24
4073 24
4074 5
4075 4
4076 0
4077 1
4078 2
4079 1
4080 6
4081 24
4082 2
4083 0
4084 1
4085 0
4086 15
4087 9
4088 0
4089 2
4090 0
4091 0
4092 1
4093 6
4094 2
4095 4
4096 12
4097 3
4098 1
4099 1
4100 5
4101 2
4102 0
4103 2
4104 2
4105 3
4106 2
4107 0
4108 4
4109 2
4110 2
4111 3
4112 24
4113 3
4114 24
4115 1
4116 2
4117 2
4118 9
4119 0
4120 0
4121 3
4122 24
4123 10
4124 6
4125 0
4126 0
4127 15
4128 12
4129 2
4130 24
4131 2
4132 0
4133 1
4134 0
4135 1
4136 2
4137 13
4138 5
4139 13
4140 0
4141 0
4142 2
4143 5
4144 0
4145 2
4146 2
4147 0
4148 17
4149 24
4150 1
4151 9
4152 2
4153 5
4154 0
4155 0
4156 0
4157 0
4158 1
4159 2
4160 24
4161 15
4162 0
4163 0
4164 0
4165 2
4166 11
4167 2
4168 7
4169 2
4170 1
4171 9
4172 0
4173 0
4174 24
4175 2
4176 2
4177 24
4178 24
4179 2
4180 22
4181 5
4182 0
4183 9
4184 0
4185 6
4186 1
4187 0
4188 2
4189 1
4190 12
4191 24
4192 7
4193 0
4194 0
4195 12
4196 6
4197 24
4198 24
4199 1
4200 10
4201 12
4202 2
4203 2
4204 1
4205 17
4206 7
4207 0
4208 2
4209 24
4210 1
4211 2
4212 13
4213 6
4214 2
4215 1
4216 1
4217 2
4218 0
4219 15
4220 2
4221 2
4222 6
4223 3
4224 2
4225 4
4226 2
4227 4
4228 2
4229 8
4230 13
4231 24
4232 22
4233 0
4234 2
4235 3
4236 5
4237 9
4238 13
4239 12
4240 0
4241 16
4242 1
4243 5
4244 24
4245 2
4246 6
4247 23
4248 24
4249 1
4250 0
4251 1
4252 4
4253 4
4254 0
4255 0
4256 10
4257 2
4258 0
4259 0
4260 1
4261 12
4262 0
4263 2
4264 2
4265 24
4266 5
4267 0
4268 24
4269 24
4270 2
4271 6
4272 0
4273 2
4274 17
4275 0
4276 15
4277 5
4278 24
4279 2
4280 2
4281 11
4282 6
4283 12
4284 2
4285 2
4286 2
4287 7
4288 2
4289 0
4290 12
4291 3
4292 0
4293 13
4294 0
4295 5
4296 1
4297 24
4298 0
4299 9
4300 2
4301 18
4302 0
4303 12
4304 0
4305 11
4306 2
4307 13
4308 1
4309 13
4310 24
4311 24
4312 24
4313 4
4314 0
4315 2
4316 5
4317 23
4318 2
4319 5
4320 18
4321 3
4322 6
4323 7
4324 20
4325 13
4326 2
4327 15
4328 17
4329 17
4330 2
4331 2
4332 24
4333 24
4334 11
4335 2
4336 17
4337 4
4338 20
4339 13
4340 2
4341 13
4342 1
4343 12
4344 2
4345 2
4346 12
4347 2
4348 1
4349 7
4350 0
4351 13
4352 1
4353 0
4354 24
4355 3
4356 2
4357 24
4358 2
4359 24
4360 0
4361 4
4362 5
4363 2
4364 1
4365 24
4366 5
4367 10
4368 22
4369 12
4370 17
4371 2
4372 0
4373 0
4374 24
4375 0
4376 2
4377 2
4378 0
4379 0
4380 6
4381 24
4382 2
4383 7
4384 1
4385 0
4386 1
4387 5
4388 12
4389 4
4390 2
4391 2
4392 24
4393 1
4394 0
4395 0
4396 2
4397 13
4398 1
4399 1
4400 13
4401 2
4402 1
4403 1
4404 6
4405 24
4406 13
4407 12
4408 16
4409 0
4410 24
4411 0
4412 17
4413 0
4414 5
4415 5
4416 0
4417 0
4418 20
4419 2
4420 2
4421 2
4422 2
4423 0
4424 12
4425 24
4426 2
4427 2
4428 3
4429 2
4430 1
4431 12
4432 24
4433 0
4434 0
4435 12
4436 1
4437 0
4438 7
4439 24
4440 8
4441 10
4442 1
4443 2
4444 0
4445 24
4446 3
4447 24
4448 15
4449 0
4450 1
4451 2
4452 22
4453 1
4454 3
4455 1
4456 13
4457 5
4458 4
4459 1
4460 2
4461 12
4462 5
4463 2
4464 0
4465 24
4466 3
4467 0
4468 0
4469 24
4470 5
4471 7
4472 2
4473 2
4474 6
4475 24
4476 1
4477 24
4478 7
4479 2
4480 1
4481 1
4482 0
4483 10
4484 12
4485 24
4486 0
4487 0
4488 2
4489 2
4490 0
4491 10
4492 24
4493 1
4494 4
4495 1
4496 1
4497 4
4498 1
4499 12
4500 0
4501 2
4502 22
4503 24
4504 0
4505 22
4506 24
4507 0
4508 1
4509 19
4510 0
4511 2
4512 0
4513 0
4514 0
4515 12
4516 5
4517 1
4518 2
4519 24
4520 24
4521 0
4522 3
4523 0
4524 2
4525 0
4526 0
4527 24
4528 2
4529 2
4530 2
4531 0
4532 24
4533 0
4534 3
4535 0
4536 1
4537 2
4538 0
4539 0
4540 0
4541 0
4542 0
4543 1
4544 2
4545 2
4546 24
4547 0
4548 0
4549 0
4550 0
4551 2
4552 0
4553 2
4554 0
4555 0
4556 0
4557 4
4558 24
4559 11
4560 4
4561 24
4562 24
4563 13
4564 4
4565 0
4566 13
4567 0
4568 1
4569 0
4570 9
4571 4
4572 6
4573 22
4574 0
4575 24
4576 9
4577 0
4578 11
4579 0
4580 2
4581 5
4582 0
4583 2
4584 0
4585 0
4586 15
4587 2
4588 3
4589 0
4590 2
4591 3
4592 24
4593 24
4594 5
4595 24
4596 3
4597 7
4598 2
4599 0
4600 3
4601 0
4602 4
4603 22
4604 6
4605 24
4606 24
4607 3
4608 2
4609 11
4610 0
4611 8
4612 3
4613 0
4614 2
4615 24
4616 2
4617 9
4618 2
4619 1
4620 5
4621 8
4622 2
4623 4
4624 24
4625 5
4626 2
4627 22
4628 0
4629 24
4630 24
4631 1
4632 14
4633 0
4634 2
4635 8
4636 13
4637 7
4638 1
4639 4
4640 6
4641 3
4642 2
4643 4
4644 7
4645 2
4646 2
4647 1
4648 2
4649 3
4650 5
4651 24
4652 2
4653 1
4654 0
4655 0
4656 0
4657 13
4658 22
4659 2
4660 0
4661 0
4662 14
4663 6
4664 21
4665 10
4666 0
4667 10
4668 4
4669 4
4670 3
4671 2
4672 1
4673 2
4674 0
4675 0
4676 24
4677 3
4678 17
4679 11
4680 2
4681 24
4682 0
4683 0
4684 0
4685 3
4686 24
4687 2
4688 0
4689 17
4690 0
4691 0
4692 9
4693 2
4694 4
4695 3
4696 20
4697 2
4698 22
4699 2
4700 5
4701 2
4702 24
4703 0
4704 2
4705 24
4706 2
4707 9
4708 0
4709 3
4710 7
4711 1
4712 0
4713 2
4714 2
4715 2
================================================
FILE: youtube-8m-ensemble/.vimrc
================================================
set tabstop=2
set shiftwidth=2
set expandtab
set autoindent
================================================
FILE: youtube-8m-ensemble/CONTRIBUTING.md
================================================
# How to contribute
We are accepting patches and contributions to this project. To set expectations,
this project is primarily intended to be a flexible starting point for
researchers working with the YouTube-8M dataset. As such, we would like to keep
it simple. We are more likely to accept small bug fixes and optimizations, and
less likely to accept patches which add significant complexity. For the latter
type of contribution, we recommend creating a Github fork of the project
instead.
If you would like to contribute, there are a few small guidelines you need to
follow.
## Contributor License Agreement
Contributions to any Google project must be accompanied by a Contributor License
Agreement. This is necessary because you own the copyright to your changes, even
after your contribution becomes part of this project. So this agreement simply
gives us permission to use and redistribute your contributions as part of the
project. Head over to to see your current
agreements on file or to sign a new one.
You generally only need to submit a CLA once, so if you've already submitted one
(even if it was for a different project), you probably don't need to do it
again.
## Code reviews
All submissions, including submissions by project members, require review. We
use GitHub pull requests for this purpose. Consult [GitHub Help] for more
information on using pull requests.
[GitHub Help]: https://help.github.com/articles/about-pull-requests/
================================================
FILE: youtube-8m-ensemble/LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: youtube-8m-ensemble/README.md
================================================
# YouTube-8M Tensorflow Starter Code
This repo contains starter code for training and evaluating machine learning
models over the [YouTube-8M](https://research.google.com/youtube8m/) dataset.
The code gives an end-to-end working example for reading the dataset, training a
TensorFlow model, and evaluating the performance of the model. Out of the box,
you can train several [model architectures](#overview-of-models) over either
frame-level or video-level features. The code can easily be extended to train
your own custom-defined models.
It is possible to train and evaluate on YouTube-8M in two ways: on Google Cloud
or on your own machine. This README provides instructions for both.
## Table of Contents
* [Running on Google's Cloud Machine Learning Platform](#running-on-googles-cloud-machine-learning-platform)
* [Requirements](#requirements)
* [Testing Locally](#testing-locally)
* [Training on the Cloud over Video-Level Features](#training-on-video-level-features)
* [Evaluation and Inference](#evaluation-and-inference)
* [Accessing Files on Google Cloud](#accessing-files-on-google-cloud)
* [Using Frame-Level Features](#using-frame-level-features)
* [Using Audio Features](#using-audio-features)
* [Running on Your Own Machine](#running-on-your-own-machine)
* [Requirements](#requirements-1)
* [Training on Video-Level Features](#training-on-video-level-features-1)
* [Evaluation and Inference](#evaluation-and-inference-1)
* [Using Frame-Level Features](#using-frame-level-features-1)
* [Using Audio Features](#using-audio-features-1)
* [Ground-Truth Label Files](#ground-truth-label-files)
* [Overview of Models](#overview-of-models)
* [Video-Level Models](#video-level-models)
* [Frame-Level Models](#frame-level-models)
* [Overview of Files](#overview-of-files)
* [Training](#training)
* [Evaluation](#evaluation)
* [Inference](#inference)
* [Misc](#misc)
* [About This Project](#about-this-project)
## Running on Google's Cloud Machine Learning Platform
### Requirements
This option requires you to have an appropriately configured Google Cloud
Platform account. To create and configure your account, please make sure you
follow the instructions [here](https://cloud.google.com/ml/docs/how-tos/getting-set-up).
If you are participating in the Google Cloud & YouTube-8M Video Understanding
Challenge hosted on [kaggle](https://www.kaggle.com/c/youtube8m), see [these instructions](https://www.kaggle.com/c/youtube8m#getting-started-with-google-cloud) instead.
Please also verify that you have Python 2.7+ and Tensorflow 1.0.0 or higher
installed by running the following commands:
```sh
python --version
python -c 'import tensorflow as tf; print(tf.__version__)'
```
### Testing Locally
All gcloud commands should be done from the directory *immediately above* the
source code. You should be able to see the source code directory if you
run 'ls'.
As you are developing your own models, you will want to test them
quickly to flush out simple problems without having to submit them to the cloud.
You can use the `gcloud beta ml local` set of commands for that.
Here is an example command line for video-level training:
```sh
gcloud beta ml local train \
--package-path=youtube-8m --module-name=youtube-8m.train -- \
--train_data_pattern='gs://youtube8m-ml/1/video_level/train/train*.tfrecord' \
--train_dir=/tmp/yt8m_train --start_new_model
```
You might want to download some training shards locally to speed things up and
allow you to work offline. The command below will copy 10 out of the 4096
training data files to the current directory.
```sh
# Downloads 55MB of data.
gsutil cp gs://us.data.yt8m.org/1/video_level/train/traina[0-9].tfrecord .
```
Once you download the files, you can point the job to them using the
'train_data_pattern' argument (i.e. instead of pointing to the "gs://..."
files, you point to the local files).
Once your model is working locally, you can scale up on the Cloud
which is described below.
### Training on the Cloud over Video-Level Features
The following commands will train a model on Google Cloud
over video-level features.
```sh
BUCKET_NAME=gs://${USER}_yt8m_train_bucket
# (One Time) Create a storage bucket to store training logs and checkpoints.
gsutil mb -l us-east1 $BUCKET_NAME
# Submit the training job.
JOB_NAME=yt8m_train_$(date +%Y%m%d_%H%M%S); gcloud --verbosity=debug beta ml jobs \
submit training $JOB_NAME \
--package-path=youtube-8m --module-name=youtube-8m.train \
--staging-bucket=$BUCKET_NAME --region=us-east1 \
--config=youtube-8m/cloudml-gpu.yaml \
-- --train_data_pattern='gs://youtube8m-ml-us-east1/1/video_level/train/train*.tfrecord' \
--train_dir=$BUCKET_NAME/yt8m_train_video_level_logistic_model
```
In the 'gsutil' command above, the 'package-path' flag refers to the directory
containing the 'train.py' script and more generally the python package which
should be deployed to the cloud worker. The module-name refers to the specific
python script which should be executed (in this case the train module).
It may take several minutes before the job starts running on Google Cloud.
When it starts you will see outputs like the following:
```
training step 270| Hit@1: 0.68 PERR: 0.52 Loss: 638.453
training step 271| Hit@1: 0.66 PERR: 0.49 Loss: 635.537
training step 272| Hit@1: 0.70 PERR: 0.52 Loss: 637.564
```
At this point you can disconnect your console by pressing "ctrl-c". The
model will continue to train indefinitely in the Cloud. Later, you can check
on its progress or halt the job by visiting the
[Google Cloud ML Jobs console](https://console.cloud.google.com/ml/jobs).
You can train many jobs at once and use tensorboard to compare their performance
visually.
```sh
tensorboard --logdir=$BUCKET_NAME --port=8080
```
Once tensorboard is running, you can access it at the following url:
[http://localhost:8080](http://localhost:8080).
If you are using Google Cloud Shell, you can instead click the Web Preview button
on the upper left corner of the Cloud Shell window and select "Preview on port 8080".
This will bring up a new browser tab with the Tensorboard view.
### Evaluation and Inference
Here's how to evaluate a model on the validation dataset:
```sh
JOB_TO_EVAL=yt8m_train_video_level_logistic_model
JOB_NAME=yt8m_eval_$(date +%Y%m%d_%H%M%S); gcloud --verbosity=debug beta ml jobs \
submit training $JOB_NAME \
--package-path=youtube-8m --module-name=youtube-8m.eval \
--staging-bucket=$BUCKET_NAME --region=us-east1 \
--config=youtube-8m/cloudml-gpu.yaml \
-- --eval_data_pattern='gs://youtube8m-ml-us-east1/1/video_level/validate/validate*.tfrecord' \
--train_dir=$BUCKET_NAME/${JOB_TO_EVAL} --run_once=True
```
And here's how to perform inference with a model on the test set:
```sh
JOB_TO_EVAL=yt8m_train_video_level_logistic_model
JOB_NAME=yt8m_inference_$(date +%Y%m%d_%H%M%S); gcloud --verbosity=debug beta ml jobs \
submit training $JOB_NAME \
--package-path=youtube-8m --module-name=youtube-8m.inference \
--staging-bucket=$BUCKET_NAME --region=us-east1 \
--config=youtube-8m/cloudml-gpu.yaml \
-- --input_data_pattern='gs://youtube8m-ml/1/video_level/test/test*.tfrecord' \
--train_dir=$BUCKET_NAME/${JOB_TO_EVAL} \
--output_file=$BUCKET_NAME/${JOB_TO_EVAL}/predictions.csv
```
Note the confusing use of 'training' in the above gcloud commands. Despite the
name, the 'training' argument really just offers a cloud hosted
python/tensorflow service. From the point of view of the Cloud Platform, there
is no distinction between our training and inference jobs. The Cloud ML platform
also offers specialized functionality for prediction with
Tensorflow models, but discussing that is beyond the scope of this readme.
Once these job starts executing you will see outputs similar to the
following for the evaluation code:
```
examples_processed: 1024 | global_step 447044 | Batch Hit@1: 0.782 | Batch PERR: 0.637 | Batch Loss: 7.821 | Examples_per_sec: 834.658
```
and the following for the inference code:
```
num examples processed: 8192 elapsed seconds: 14.85
```
### Accessing Files on Google Cloud
You can browse the storage buckets you created on Google Cloud, for example, to
access the trained models, prediction CSV files, etc. by visiting the
[Google Cloud storage browser](https://console.cloud.google.com/storage/browser).
Alternatively, you can use the 'gsutil' command to download the files directly.
For example, to download the output of the inference code from the previous
section to your local machine, run:
```
gsutil cp $BUCKET_NAME/${JOB_TO_EVAL}/predictions.csv .
```
### Using Frame-Level Features
Append
```sh
--frame_features=True --model=FrameLevelLogisticModel --feature_names="rgb" \
--feature_sizes="1024" --batch_size=128 \
--train_dir=$BUCKET_NAME/yt8m_train_frame_level_logistic_model
```
to the 'gcloud' commands given above, and change 'video_level' in paths to
'frame_level'. Here is a sample command to kick-off a frame-level job:
```sh
JOB_NAME=yt8m_train_$(date +%Y%m%d_%H%M%S); gcloud --verbosity=debug beta ml jobs \
submit training $JOB_NAME \
--package-path=youtube-8m --module-name=youtube-8m.train \
--staging-bucket=$BUCKET_NAME --region=us-east1 \
--config=youtube-8m/cloudml-gpu.yaml \
-- --train_data_pattern='gs://youtube8m-ml-us-east1/1/frame_level/train/train*.tfrecord' \
--frame_features=True --model=FrameLevelLogisticModel --feature_names="rgb" \
--feature_sizes="1024" --batch_size=128 \
--train_dir=$BUCKET_NAME/yt8m_train_frame_level_logistic_model
```
The 'FrameLevelLogisticModel' is designed to provide equivalent results to a
logistic model trained over the video-level features. Please look at the
'video_level_models.py' or 'frame_level_models.py' files to see how to implement
your own models.
### Using Audio Features
The feature files (both Frame-Level and Video-Level) contain two sets of
features: 1) visual and 2) audio. The code defaults to using the visual
features only, but it is possible to use audio features instead of (or besides)
visual features. To specify the (combination of) features to use you must set
`--feature_names` and `--feature_sizes` flags. The visual and audio features are
called 'rgb' and 'audio' and have 1024 and 128 dimensions, respectively.
The two flags take a comma-separated list of values in string. For example, to
use audio-visual Video-Level features the flags must be set as follows:
```
--feature_names="mean_rgb, mean_audio" --feature_sizes="1024, 128"
```
Similarly, to use audio-visual Frame-Level features use:
```
--feature_names="rgb, audio" --feature_sizes="1024, 128"
```
**NOTE:** Make sure the set of features and the order in which the appear in the
lists provided to the two flags above match. Also, the order must match when
running training, evaluation, or inference.
## Running on Your Own Machine
### Requirements
The starter code requires Tensorflow. If you haven't installed it yet, follow
the instructions on [tensorflow.org](https://www.tensorflow.org/install/).
This code has been tested with Tensorflow 1.0.0. Going forward, we will continue
to target the latest released version of Tensorflow.
Please verify that you have Python 2.7+ and Tensorflow 1.0.0 or higher
installed by running the following commands:
```sh
python --version
python -c 'import tensorflow as tf; print(tf.__version__)'
```
You can find complete instructions for downloading the dataset on the
[YouTube-8M website](https://research.google.com/youtube8m/download.html).
We recommend downloading the smaller video-level features dataset first when
getting started. To do that, run:
```
mkdir -p features; cd features
curl data.yt8m.org/download.py | partition=1/video_level/train mirror=us python
```
This will download the full set of video level features, which takes up 31GB
of space.
If you are located outside of North America, you should change the flag 'mirror'
to 'eu' for Europe or 'asia' for Asia to speed up the transfer of the files.
Change 'train' to 'validate'/'test' and re-run the command to download the
other splits of the dataset.
Change 'video_level' to 'frame_level' to download the frame-level features. The
complete frame-level features take about 1.71TB of space. You can set the
environment variable 'shard' to 'm,n' to download only m/n-th of the data. For
example, to download 1/100-th of the frame-level features from the training set,
run:
```
curl data.yt8m.org/download.py | shard=1,100 partition=1/frame_level/train mirror=us python
```
### Training on Video-Level Features
To start training a logistic model on the video-level features, run
```sh
MODEL_DIR=/tmp/yt8m
python train.py --train_data_pattern='/path/to/features/train*.tfrecord' --train_dir=$MODEL_DIR/video_level_logistic_model
```
Since the dataset is sharded into 4096 individual files, we use a wildcard (\*)
to represent all of those files.
By default, the training code will frequently write _checkpoint_ files (i.e.
values of all trainable parameters, at the current training iteration). These
will be written to the `--train_dir`. If you re-use a `--train_dir`, the trainer
will first restore the latest checkpoint written in that directory. This only
works if the architecture of the checkpoint matches the graph created by the
training code. If you are in active development/debugging phase, consider
adding `--start_new_model` flag to your run configuration.
### Evaluation and Inference
To evaluate the model, run
```sh
python eval.py --eval_data_pattern='/path/to/features/validate*.tfrecord' --train_dir=$MODEL_DIR/video_level_logistic_model --run_once=True
```
As the model is training or evaluating, you can view the results on tensorboard
by running
```sh
tensorboard --logdir=$MODEL_DIR
```
and navigating to http://localhost:6006 in your web browser.
When you are happy with your model, you can generate a csv file of predictions
from it by running
```sh
python inference.py --output_file=$MODEL_DIR/video_level_logistic_model/predictions.csv --input_data_pattern='/path/to/features/test*.tfrecord' --train_dir=$MODEL_DIR/video_level_logistic_model
```
This will output the top 20 predicted labels from the model for every example
to 'predictions.csv'.
### Using Frame-Level Features
Follow the same instructions as above, appending
`--frame_features=True --model=FrameLevelLogisticModel --feature_names="rgb"
--feature_sizes="1024" --train_dir=$MODEL_DIR/frame_level_logistic_model`
for the 'train.py', 'eval.py', and 'inference.py' scripts.
The 'FrameLevelLogisticModel' is designed to provide equivalent results to a
logistic model trained over the video-level features. Please look at the
'models.py' file to see how to implement your own models.
### Using Audio Features
See [Using Audio Features](#using-audio-features) section above.
### Ground-Truth Label Files
We also provide CSV files containing the ground-truth label information of the
'train' and 'validation' partitions of the dataset. These files can be
downloaded using 'gsutil' command:
```
gsutil cp gs://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv /destination/folder/
gsutil cp gs://us.data.yt8m.org/1/ground_truth_labels/validate_labels.csv /destination/folder/
```
or directly using the following links:
* [http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv](http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv)
* [http://us.data.yt8m.org/1/ground_truth_labels/validate_labels.csv](http://us.data.yt8m.org/1/ground_truth_labels/validate_labels.csv)
Each line in the files starts with the video id and is followed by the list of
ground-truth labels corresponding to that video. For example, for a video with
id 'VIDEO_ID' and two lables 'LABLE1' and 'LABEL2' we store the following line:
```
VIDEO_ID,LABEL1 LABEL2
```
## Overview of Models
This sample code contains implementations of the models given in the
[YouTube-8M technical report](https://arxiv.org/abs/1609.08675).
### Video-Level Models
* `LogisticModel`: Linear projection of the output features into the label
space, followed by a sigmoid function to convert logit
values to probabilities.
* `MoeModel`: A per-class softmax distribution over a configurable number of
logistic classifiers. One of the classifiers in the mixture
is not trained, and always predicts 0.
### Frame-Level Models
* `LstmModel`: Processes the features for each frame using a multi-layered
LSTM neural net. The final internal state of the LSTM
is input to a video-level model for classification. Note that
you will need to change the learning rate to 0.001 when using
this model.
* `DbofModel`: Projects the features for each frame into a higher dimensional
'clustering' space, pools across frames in that space, and then
uses a video-level model to classify the now aggregated features.
* `FrameLevelLogisticModel`: Equivalent to 'LogisticModel', but performs
average-pooling on the fly over frame-level
features rather than using pre-aggregated features.
## Overview of Files
### Training
* `train.py`: The primary script for training models.
* `losses.py`: Contains definitions for loss functions.
* `models.py`: Contains the base class for defining a model.
* `video_level_models.py`: Contains definitions for models that take
aggregated features as input.
* `frame_level_models.py`: Contains definitions for models that take frame-
level features as input.
* `model_util.py`: Contains functions that are of general utility for
implementing models.
* `readers.py`: Contains definitions for the Video dataset and Frame
dataset readers.
### Evaluation
* `eval.py`: The primary script for evaluating models.
* `eval_util.py`: Provides a class that calculates all evaluation metrics.
* `average_precision_calculator.py`: Functions for calculating
average precision.
* `mean_average_precision_calculator.py`: Functions for calculating mean
average precision.
### Inference
* `inference.py`: Generates an output file containing predictions of
the model over a set of videos.
### Misc
* `README.md`: This documentation.
* `utils.py`: Common functions.
## About This Project
This project is meant help people quickly get started working with the
[YouTube-8M](https://research.google.com/youtube8m/) dataset.
This is not an official Google product.
================================================
FILE: youtube-8m-ensemble/__init__.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
================================================
FILE: youtube-8m-ensemble/all_ensemble_models/.vimrc
================================================
set tabstop=2
set shiftwidth=2
set expandtab
set autoindent
================================================
FILE: youtube-8m-ensemble/all_ensemble_models/__init__.py
================================================
from logistic_model import *
from moe_model import *
from attention_moe_model import *
from attention_moe_matrix_model import *
from input_moe_model import *
from linear_regression_model import *
from matrix_regression_model import *
from attention_matrix_model import *
from attention_linear_model import *
from attention_linmatrix_model import *
from attention_rectified_linear_model import *
from nonunit_matrix_regression_model import *
from mean_model import *
from deep_combine_chain_model import *
================================================
FILE: youtube-8m-ensemble/all_ensemble_models/attention_linear_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class AttentionLinearModel(models.BaseModel):
def create_model(self,
model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
sub_scope="",
original_input=None,
**unused_params):
num_methods = model_input.get_shape().as_list()[-1]
num_features = model_input.get_shape().as_list()[-2]
num_mixtures = FLAGS.moe_num_mixtures
# gating coefficients
original_input = tf.nn.l2_normalize(original_input, dim=1)
mean_output = tf.reduce_mean(model_input, axis=2)
## batch_size x moe_num_mixtures
gate_activations = slim.fully_connected(
tf.concat([original_input, mean_output], axis=1),
num_mixtures,
activation_fn=tf.nn.softmax,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+sub_scope)
# matrix
weight_var = tf.get_variable("ensemble_weight",
shape=[num_mixtures, num_methods],
regularizer=slim.l2_regularizer(l2_penalty))
# weight
gated_weight = tf.einsum("ij,jk->ik", gate_activations, weight_var)
weight = tf.nn.softmax(gated_weight)
# weighted output
output = tf.einsum("ik,ijk->ij", weight, model_input)
return {"predictions": output}
================================================
FILE: youtube-8m-ensemble/all_ensemble_models/attention_linmatrix_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class AttentionLinmatrixModel(models.BaseModel):
def create_model(self,
model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
sub_scope="",
original_input=None,
**unused_params):
num_methods = model_input.get_shape().as_list()[-1]
num_features = model_input.get_shape().as_list()[-2]
num_mixtures = FLAGS.moe_num_mixtures
attention_matrix_rank = FLAGS.attention_matrix_rank
# gating coefficients
original_input = tf.nn.l2_normalize(original_input, dim=1)
mean_output = tf.reduce_mean(model_input, axis=2)
## batch_size x moe_num_mixtures
gate_activations = slim.fully_connected(
tf.concat([original_input, mean_output], axis=1),
num_mixtures,
activation_fn=tf.nn.softmax,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+sub_scope)
# matrix
b = tf.get_variable("ensemble_bias", shape=[num_mixtures,1,1])
weight_var = tf.get_variable("ensemble_weight",
shape=[num_mixtures, 1, num_methods],
regularizer=slim.l2_regularizer(l2_penalty))
weight_x = tf.get_variable("ensemble_weightx",
shape=[num_mixtures, num_features, attention_matrix_rank],
regularizer=slim.l2_regularizer(l2_penalty))
weight_y = tf.get_variable("ensemble_weighty",
shape=[num_mixtures, attention_matrix_rank, num_methods],
regularizer=slim.l2_regularizer(l2_penalty))
## moe_num_mixtures x num_features x num_methods
weight_xy = tf.einsum("ijl,ilk->ijk", weight_x, weight_y) + weight_var + b
# weight
gated_weight_xy = tf.einsum("ij,jkl->ikl", gate_activations, weight_xy)
weight = tf.nn.softmax(gated_weight_xy)
# weighted output
output = tf.reduce_sum(weight * model_input, axis=2)
return {"predictions": output}
================================================
FILE: youtube-8m-ensemble/all_ensemble_models/attention_matrix_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class AttentionMatrixModel(models.BaseModel):
def create_model(self,
model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
sub_scope="",
original_input=None,
**unused_params):
num_methods = model_input.get_shape().as_list()[-1]
num_features = model_input.get_shape().as_list()[-2]
num_mixtures = FLAGS.moe_num_mixtures
attention_matrix_rank = FLAGS.attention_matrix_rank
# gating coefficients
original_input = tf.nn.l2_normalize(original_input, dim=1)
mean_output = tf.reduce_mean(model_input, axis=2)
## batch_size x moe_num_mixtures
gate_activations = slim.fully_connected(
tf.concat([original_input, mean_output], axis=1),
num_mixtures,
activation_fn=tf.nn.softmax,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+sub_scope)
# matrix
weight_x = tf.get_variable("ensemble_weightx",
shape=[num_mixtures, num_features, attention_matrix_rank],
regularizer=slim.l2_regularizer(l2_penalty))
weight_y = tf.get_variable("ensemble_weighty",
shape=[num_mixtures, attention_matrix_rank, num_methods],
regularizer=slim.l2_regularizer(l2_penalty))
## moe_num_mixtures x num_features x num_methods
weight_xy = tf.einsum("ijl,ilk->ijk", weight_x, weight_y)
# weight
gated_weight_xy = tf.einsum("ij,jkl->ikl", gate_activations, weight_xy)
weight = tf.nn.softmax(gated_weight_xy)
# weighted output
output = tf.reduce_sum(weight * model_input, axis=2)
return {"predictions": output}
================================================
FILE: youtube-8m-ensemble/all_ensemble_models/attention_moe_matrix_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class AttentionMoeMatrixModel(models.BaseModel):
def create_model(self,
model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
sub_scope="",
original_input=None,
**unused_params):
num_relu = FLAGS.attention_relu_cells
num_methods = model_input.get_shape().as_list()[-1]
num_features = model_input.get_shape().as_list()[-2]
num_mixtures = FLAGS.moe_num_mixtures
attention_matrix_rank = FLAGS.attention_matrix_rank
# gating coefficients
mean_input = tf.reduce_mean(model_input, axis=2)
std_input = tf.reduce_sum(
tf.square(model_input - tf.expand_dims(mean_input, dim=2)),
axis=2) / (num_methods - 1)
# relu
original_relu = self.relu(original_input, num_relu, sub_scope="origin")
mean_relu = self.relu(mean_input, num_relu, sub_scope="mean")
std_relu = self.relu(std_input, num_relu, sub_scope="std")
# normalize
original_relu = tf.nn.l2_normalize(original_relu)
mean_relu = tf.nn.l2_normalize(mean_relu)
std_relu = tf.nn.l2_normalize(std_relu)
## batch_size x moe_num_mixtures
gate_activations = slim.fully_connected(
tf.concat([original_relu, mean_relu, std_relu], axis=1),
num_mixtures,
activation_fn=tf.nn.softmax,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+sub_scope)
# matrix
weight_x = tf.get_variable("ensemble_weightx",
shape=[num_mixtures, num_features, attention_matrix_rank],
regularizer=slim.l2_regularizer(l2_penalty))
weight_y = tf.get_variable("ensemble_weighty",
shape=[num_mixtures, attention_matrix_rank, num_methods],
regularizer=slim.l2_regularizer(l2_penalty))
## moe_num_mixtures x num_features x num_methods
weight_xy = tf.einsum("ijl,ilk->ijk", weight_x, weight_y)
# weight
gated_weight_xy = tf.einsum("ij,jkl->ikl", gate_activations, weight_xy)
weight = tf.nn.softmax(gated_weight_xy)
# weighted output
output = tf.reduce_sum(weight * model_input, axis=2)
return {"predictions": output}
def relu(self, model_input, relu_cells,
l2_penalty=1e-8, sub_scope=""):
sub_activation = slim.fully_connected(
model_input,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="relu-"+sub_scope)
return sub_activation
================================================
FILE: youtube-8m-ensemble/all_ensemble_models/attention_moe_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class AttentionMoeModel(models.BaseModel):
def create_model(self,
model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
sub_scope="",
original_input=None,
**unused_params):
num_relu = FLAGS.attention_relu_cells
num_methods = model_input.get_shape().as_list()[-1]
num_features = model_input.get_shape().as_list()[-2]
original_input = tf.nn.l2_normalize(original_input, dim=1)
model_input_list = tf.unstack(model_input, axis=2)
relu_units = [self.relu(original_input, num_relu, sub_scope="input")]
i = 0
for mi in model_input_list:
relu_units.append(self.relu(mi, num_relu, sub_scope="sub"+str(i)))
i += 1
gate_activations = slim.fully_connected(
tf.concat(relu_units, axis=1),
num_methods,
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gate")
gate = tf.nn.softmax(gate_activations)
output = tf.einsum("ijk,ik->ij", model_input, gate)
return {"predictions": output}
def relu(self, model_input, relu_cells,
l2_penalty=1e-8, sub_scope=""):
sub_activation = slim.fully_connected(
model_input,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="relu-"+sub_scope)
return sub_activation
================================================
FILE: youtube-8m-ensemble/all_ensemble_models/attention_rectified_linear_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class AttentionRectifiedLinearModel(models.BaseModel):
def create_model(self,
model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
sub_scope="",
original_input=None,
**unused_params):
num_methods = model_input.get_shape().as_list()[-1]
num_features = model_input.get_shape().as_list()[-2]
num_mixtures = FLAGS.moe_num_mixtures
# gating coefficients
original_input = tf.nn.l2_normalize(original_input, dim=1)
mean_output = tf.reduce_mean(model_input, axis=2)
## batch_size x moe_num_mixtures
gate_activations = slim.fully_connected(
tf.concat([original_input, mean_output], axis=1),
num_mixtures,
activation_fn=tf.nn.softmax,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+sub_scope)
# matrix
weight_var = tf.get_variable("ensemble_weight",
shape=[num_mixtures, num_methods],
regularizer=slim.l2_regularizer(l2_penalty))
# weight
gated_weight = tf.einsum("ij,jk->ik", gate_activations, weight_var)
rl_gated_weight = tf.nn.relu(gated_weight) + 1e-9
sum_gated_weight = tf.reduce_sum(rl_gated_weight, axis=1, keep_dims=True)
weight = rel_gated_weight / sum_gated_weight
# weighted output
output = tf.einsum("ik,ijk->ij", weight, model_input)
return {"predictions": output}
================================================
FILE: youtube-8m-ensemble/all_ensemble_models/deep_combine_chain_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class DeepCombineChainModel(models.BaseModel):
def create_model(self,
model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
sub_scope="",
original_input=None,
**unused_params):
num_methods = model_input.get_shape().as_list()[-1]
num_features = model_input.get_shape().as_list()[-2]
num_mixtures = FLAGS.moe_num_mixtures
attention_matrix_rank = FLAGS.attention_matrix_rank
relu_cells = FLAGS.deep_chain_relu_cells
# mean_output
mean_output = tf.reduce_mean(model_input, axis=2)
mean_relu = slim.fully_connected(
mean_output,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
reuse=True,
scope=sub_scope+"transform")
permuted_input = tf.transpose(model_input, perm=[0,2,1])
input_relu = slim.fully_connected(
permuted_input,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
reuse=True,
scope=sub_scope+"transform")
input_relu_list = tf.unstack(input_relu, axis=1)
predictions = []
discounted_prediction = mean_output
prev_relu = mean_relu
for i in xrange(num_methods):
cur_relu = input_relu_list[i]
cur_prediction = self.sub_moe(
model_input=tf.concat([prev_relu, cur_relu], axis=1),
vocab_size=vocab_size,
sub_scope="predict-%d"%i)
discounted_prediction = (discounted_prediction + cur_prediction) / 2
# if not the last one
if i + 1 < num_methods:
predictions.append(cur_prediction)
new_relu = slim.fully_connected(
cur_prediction,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
reuse=True,
scope=sub_scope+"compress")
final_prediction = discounted_prediction
support_prediction = tf.reduce_mean(tf.stack(predictions, axis=0), axis=0)
return {"predictions": final_prediction,
"support_predictions": support_prediction}
def sub_moe(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="",
**unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
================================================
FILE: youtube-8m-ensemble/all_ensemble_models/input_moe_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class InputMoeModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self,
model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
sub_scope="",
original_input=None,
**unused_params):
num_methods = model_input.get_shape().as_list()[-1]
num_features = model_input.get_shape().as_list()[-2]
original_input = tf.nn.l2_normalize(original_input, dim=1)
gate_activations = slim.fully_connected(
original_input,
num_methods,
activation_fn=tf.nn.softmax,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+sub_scope)
output = tf.einsum("ijk,ik->ij", model_input, gate_activations)
return {"predictions": output}
================================================
FILE: youtube-8m-ensemble/all_ensemble_models/linear_regression_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class LinearRegressionModel(models.BaseModel):
"""Logistic model with L2 regularization."""
def create_model(self, model_input, vocab_size, l2_penalty=1e-8, original_input=None, **unused_params):
"""Creates a linear regression model.
Args:
model_input: 'batch' x 'num_features' x 'num_methods' matrix of input features.
vocab_size: The number of classes in the dataset.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes."""
num_methods = model_input.get_shape().as_list()[-1]
weight = tf.get_variable("ensemble_weight",
shape=[num_methods],
regularizer=slim.l2_regularizer(l2_penalty))
weight = tf.nn.softmax(weight)
output = tf.einsum("ijk,k->ij", model_input, weight)
return {"predictions": output}
================================================
FILE: youtube-8m-ensemble/all_ensemble_models/logistic_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class LogisticModel(models.BaseModel):
"""Logistic model with L2 regularization."""
def create_model(self, model_input, vocab_size, l2_penalty=1e-8, original_input=None, **unused_params):
"""Creates a logistic model.
Args:
model_input: 'batch' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes."""
output = slim.fully_connected(
model_input, vocab_size, activation_fn=tf.nn.sigmoid,
weights_regularizer=slim.l2_regularizer(l2_penalty))
return {"predictions": output}
================================================
FILE: youtube-8m-ensemble/all_ensemble_models/matrix_regression_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class MatrixRegressionModel(models.BaseModel):
"""Logistic model with L2 regularization."""
def create_model(self, model_input, vocab_size, l2_penalty=1e-8, original_input=None, **unused_params):
"""Creates a matrix regression model.
Args:
model_input: 'batch' x 'num_features' x 'num_methods' matrix of input features.
vocab_size: The number of classes in the dataset.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes."""
num_features = model_input.get_shape().as_list()[-2]
num_methods = model_input.get_shape().as_list()[-1]
weight1d = tf.get_variable("ensemble_weight1d",
shape=[num_methods],
regularizer=slim.l2_regularizer(l2_penalty))
weight2d = tf.get_variable("ensemble_weight2d",
shape=[num_features, num_methods],
regularizer=slim.l2_regularizer(10 * l2_penalty))
weight = tf.nn.softmax(tf.einsum("ij,j->ij", weight2d, weight1d), dim=-1)
output = tf.einsum("ijk,jk->ij", model_input, weight)
return {"predictions": output}
================================================
FILE: youtube-8m-ensemble/all_ensemble_models/mean_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class MeanModel(models.BaseModel):
"""Mean model."""
def create_model(self, model_input, **unused_params):
"""Creates a logistic model.
model_input: 'batch' x 'num_features' x 'num_methods' matrix of input features.
vocab_size: The number of classes in the dataset.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes."""
output = tf.reduce_mean(model_input, axis=2)
return {"predictions": output}
================================================
FILE: youtube-8m-ensemble/all_ensemble_models/moe_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class MoeModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self,
model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
sub_scope="",
original_input=None,
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_methods = model_input.get_shape().as_list()[-1]
num_features = model_input.get_shape().as_list()[-2]
flat_input = tf.reshape(model_input, shape=[-1,num_features * num_methods])
tensor_weight = tf.get_variable("tensor_weight",
shape=[num_features, num_methods, num_methods],
regularizer=slim.l2_regularizer(l2_penalty))
tensor_bias = tf.get_variable("tensor_bias",
shape=[num_features, num_methods],
initializer=tf.zeros_initializer(),
regularizer=slim.l2_regularizer(l2_penalty))
gate_activations = tf.einsum("ijk,jkl->ijl", model_input, tensor_weight) \
+ tf.expand_dims(tensor_bias, dim=0)
output = tf.reduce_sum(model_input * tf.nn.softmax(gate_activations), axis=2)
return {"predictions": output}
================================================
FILE: youtube-8m-ensemble/all_ensemble_models/nonunit_matrix_regression_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class NonunitMatrixRegressionModel(models.BaseModel):
"""Logistic model with L2 regularization."""
def create_model(self, model_input, vocab_size, l2_penalty=1e-8, original_input=None, epsilon=1e-5, **unused_params):
"""Creates a non-unified matrix regression model.
Args:
model_input: 'batch' x 'num_features' x 'num_methods' matrix of input features.
vocab_size: The number of classes in the dataset.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes."""
num_features = model_input.get_shape().as_list()[-2]
num_methods = model_input.get_shape().as_list()[-1]
log_model_input = tf.stop_gradient(tf.log((epsilon + model_input) / (1.0 + epsilon - model_input)))
weight = tf.get_variable("ensemble_weight",
shape=[num_features, num_methods],
regularizer=slim.l2_regularizer(l2_penalty))
weight = tf.nn.softmax(weight)
output = tf.nn.sigmoid(tf.einsum("ijk,jk->ij", log_model_input, weight))
return {"predictions": output}
================================================
FILE: youtube-8m-ensemble/average_precision_calculator.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Calculate or keep track of the interpolated average precision.
It provides an interface for calculating interpolated average precision for an
entire list or the top-n ranked items. For the definition of the
(non-)interpolated average precision:
http://trec.nist.gov/pubs/trec15/appendices/CE.MEASURES06.pdf
Example usages:
1) Use it as a static function call to directly calculate average precision for
a short ranked list in the memory.
```
import random
p = np.array([random.random() for _ in xrange(10)])
a = np.array([random.choice([0, 1]) for _ in xrange(10)])
ap = average_precision_calculator.AveragePrecisionCalculator.ap(p, a)
```
2) Use it as an object for long ranked list that cannot be stored in memory or
the case where partial predictions can be observed at a time (Tensorflow
predictions). In this case, we first call the function accumulate many times
to process parts of the ranked list. After processing all the parts, we call
peek_interpolated_ap_at_n.
```
p1 = np.array([random.random() for _ in xrange(5)])
a1 = np.array([random.choice([0, 1]) for _ in xrange(5)])
p2 = np.array([random.random() for _ in xrange(5)])
a2 = np.array([random.choice([0, 1]) for _ in xrange(5)])
# interpolated average precision at 10 using 1000 break points
calculator = average_precision_calculator.AveragePrecisionCalculator(10)
calculator.accumulate(p1, a1)
calculator.accumulate(p2, a2)
ap3 = calculator.peek_ap_at_n()
```
"""
import heapq
import random
import numbers
import numpy
class AveragePrecisionCalculator(object):
"""Calculate the average precision and average precision at n."""
def __init__(self, top_n=None):
"""Construct an AveragePrecisionCalculator to calculate average precision.
This class is used to calculate the average precision for a single label.
Args:
top_n: A positive Integer specifying the average precision at n, or
None to use all provided data points.
Raises:
ValueError: An error occurred when the top_n is not a positive integer.
"""
if not ((isinstance(top_n, int) and top_n >= 0) or top_n is None):
raise ValueError("top_n must be a positive integer or None.")
self._top_n = top_n # average precision at n
self._total_positives = 0 # total number of positives have seen
self._heap = [] # max heap of (prediction, actual)
@property
def heap_size(self):
"""Gets the heap size maintained in the class."""
return len(self._heap)
@property
def num_accumulated_positives(self):
"""Gets the number of positive samples that have been accumulated."""
return self._total_positives
def accumulate(self, predictions, actuals, num_positives=None):
"""Accumulate the predictions and their ground truth labels.
After the function call, we may call peek_ap_at_n to actually calculate
the average precision.
Note predictions and actuals must have the same shape.
Args:
predictions: a list storing the prediction scores.
actuals: a list storing the ground truth labels. Any value
larger than 0 will be treated as positives, otherwise as negatives.
num_positives = If the 'predictions' and 'actuals' inputs aren't complete,
then it's possible some true positives were missed in them. In that case,
you can provide 'num_positives' in order to accurately track recall.
Raises:
ValueError: An error occurred when the format of the input is not the
numpy 1-D array or the shape of predictions and actuals does not match.
"""
if len(predictions) != len(actuals):
raise ValueError("the shape of predictions and actuals does not match.")
if not num_positives is None:
if not isinstance(num_positives, numbers.Number) or num_positives < 0:
raise ValueError("'num_positives' was provided but it wan't a nonzero number.")
if not num_positives is None:
self._total_positives += num_positives
else:
self._total_positives += numpy.size(numpy.where(actuals > 0))
topk = self._top_n
heap = self._heap
for i in range(numpy.size(predictions)):
if topk is None or len(heap) < topk:
heapq.heappush(heap, (predictions[i], actuals[i]))
else:
if predictions[i] > heap[0][0]: # heap[0] is the smallest
heapq.heappop(heap)
heapq.heappush(heap, (predictions[i], actuals[i]))
def clear(self):
"""Clear the accumulated predictions."""
self._heap = []
self._total_positives = 0
def peek_ap_at_n(self):
"""Peek the non-interpolated average precision at n.
Returns:
The non-interpolated average precision at n (default 0).
If n is larger than the length of the ranked list,
the average precision will be returned.
"""
if self.heap_size <= 0:
return 0
predlists = numpy.array(list(zip(*self._heap)))
ap = self.ap_at_n(predlists[0],
predlists[1],
n=self._top_n,
total_num_positives=self._total_positives)
return ap
@staticmethod
def ap(predictions, actuals):
"""Calculate the non-interpolated average precision.
Args:
predictions: a numpy 1-D array storing the sparse prediction scores.
actuals: a numpy 1-D array storing the ground truth labels. Any value
larger than 0 will be treated as positives, otherwise as negatives.
Returns:
The non-interpolated average precision at n.
If n is larger than the length of the ranked list,
the average precision will be returned.
Raises:
ValueError: An error occurred when the format of the input is not the
numpy 1-D array or the shape of predictions and actuals does not match.
"""
return AveragePrecisionCalculator.ap_at_n(predictions,
actuals,
n=None)
@staticmethod
def ap_at_n(predictions, actuals, n=20, total_num_positives=None):
"""Calculate the non-interpolated average precision.
Args:
predictions: a numpy 1-D array storing the sparse prediction scores.
actuals: a numpy 1-D array storing the ground truth labels. Any value
larger than 0 will be treated as positives, otherwise as negatives.
n: the top n items to be considered in ap@n.
total_num_positives : (optionally) you can specify the number of total
positive
in the list. If specified, it will be used in calculation.
Returns:
The non-interpolated average precision at n.
If n is larger than the length of the ranked list,
the average precision will be returned.
Raises:
ValueError: An error occurred when
1) the format of the input is not the numpy 1-D array;
2) the shape of predictions and actuals does not match;
3) the input n is not a positive integer.
"""
if len(predictions) != len(actuals):
raise ValueError("the shape of predictions and actuals does not match.")
if n is not None:
if not isinstance(n, int) or n <= 0:
raise ValueError("n must be 'None' or a positive integer."
" It was '%s'." % n)
ap = 0.0
predictions = numpy.array(predictions)
actuals = numpy.array(actuals)
# add a shuffler to avoid overestimating the ap
predictions, actuals = AveragePrecisionCalculator._shuffle(predictions,
actuals)
sortidx = sorted(
range(len(predictions)),
key=lambda k: predictions[k],
reverse=True)
if total_num_positives is None:
numpos = numpy.size(numpy.where(actuals > 0))
else:
numpos = total_num_positives
if numpos == 0:
return 0
if n is not None:
numpos = min(numpos, n)
delta_recall = 1.0 / numpos
poscount = 0.0
# calculate the ap
r = len(sortidx)
if n is not None:
r = min(r, n)
for i in range(r):
if actuals[sortidx[i]] > 0:
poscount += 1
ap += poscount / (i + 1) * delta_recall
return ap
@staticmethod
def _shuffle(predictions, actuals):
random.seed(0)
suffidx = random.sample(range(len(predictions)), len(predictions))
predictions = predictions[suffidx]
actuals = actuals[suffidx]
return predictions, actuals
@staticmethod
def _zero_one_normalize(predictions, epsilon=1e-7):
"""Normalize the predictions to the range between 0.0 and 1.0.
For some predictions like SVM predictions, we need to normalize them before
calculate the interpolated average precision. The normalization will not
change the rank in the original list and thus won't change the average
precision.
Args:
predictions: a numpy 1-D array storing the sparse prediction scores.
epsilon: a small constant to avoid denominator being zero.
Returns:
The normalized prediction.
"""
denominator = numpy.max(predictions) - numpy.min(predictions)
ret = (predictions - numpy.min(predictions)) / numpy.max(denominator,
epsilon)
return ret
================================================
FILE: youtube-8m-ensemble/check_distillation.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for evaluating Tensorflow models on the YouTube-8M dataset."""
import time
import numpy as np
import eval_util
import losses
import ensemble_level_models
import readers
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import utils
FLAGS = flags.FLAGS
if __name__ == "__main__":
# Dataset flags.
flags.DEFINE_string(
"eval_data_patterns", "",
"File globs defining the evaluation dataset in tensorflow.SequenceExample format.")
flags.DEFINE_string(
"input_data_pattern", None,
"File globs for original model input.")
flags.DEFINE_string("feature_names", "predictions", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "4716", "Length of the feature vectors.")
# Model flags.
flags.DEFINE_integer("batch_size", 1024,
"How many examples to process per batch.")
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def get_input_evaluation_tensors(reader,
data_pattern,
batch_size=256):
logging.info("Using batch size of " + str(batch_size) + " for evaluation.")
with tf.name_scope("eval_input"):
files = gfile.Glob(data_pattern)
if not files:
print data_pattern, files
raise IOError("Unable to find the evaluation files.")
logging.info("number of evaluation files: " + str(len(files)))
files.sort()
filename_queue = tf.train.string_input_producer(
files, shuffle=False, num_epochs=1)
eval_data = reader.prepare_reader(filename_queue)
return tf.train.batch(
eval_data,
batch_size=batch_size,
capacity=3 * batch_size,
allow_smaller_final_batch=True,
enqueue_many=True)
def build_graph(all_readers,
input_reader,
input_data_pattern,
all_eval_data_patterns,
batch_size=256):
original_video_id, original_input, unused_labels_batch, unused_num_frames = (
get_input_evaluation_tensors(
input_reader,
input_data_pattern,
batch_size=batch_size))
video_id_equal_tensors = []
model_input_tensor = None
input_distance_tensors = []
model_label_tensor = tf.cast(unused_labels_batch, dtype=tf.float32)
label_distance_tensors = []
for reader, data_pattern in zip(all_readers, all_eval_data_patterns):
video_id, model_input_raw, labels_batch, unused_num_frames = (
get_input_evaluation_tensors(
reader,
data_pattern,
batch_size=batch_size))
video_id_equal_tensors.append(tf.reduce_sum(tf.cast(tf.not_equal(original_video_id, video_id), dtype=tf.float32)))
input_distance_tensors.append(tf.reduce_mean(tf.reduce_sum(tf.square(original_input - model_input_raw), axis=1)))
labels_batch = tf.cast(labels_batch, dtype=tf.float32)
x = model_input_raw
y = labels_batch
ce = - y * tf.log(x + 1e-7) - (1.0 - y) * tf.log(1.0 + 1e-7 - x)
label_distance_tensors.append(tf.reduce_mean(tf.reduce_sum(ce, axis=1)))
video_id_equal_tensor = tf.stack(video_id_equal_tensors)
input_distance_tensor = tf.stack(input_distance_tensors)
label_distance_tensor = tf.stack(label_distance_tensors)
tf.add_to_collection("model_input", model_input_tensor)
tf.add_to_collection("video_id_equal", video_id_equal_tensor)
tf.add_to_collection("input_distance", input_distance_tensor)
tf.add_to_collection("label_distance", label_distance_tensor)
def check_loop(model_input, video_id_equal, input_distance, label_distance, all_patterns):
with tf.Session() as sess:
sess.run([tf.local_variables_initializer()])
# Start the queue runners.
coord = tf.train.Coordinator()
try:
threads = []
for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
threads.extend(qr.create_threads(
sess, coord=coord, daemon=True,
start=True))
examples_processed = 0
while not coord.should_stop():
batch_start_time = time.time()
model_input_val, video_id_equal_val, input_distance_val, label_distance_val = sess.run([model_input, video_id_equal, input_distance, label_distance])
print "model_input.max", np.max(model_input_val)
print "model_input.min", np.min(model_input_val)
print "input_distance_val", input_distance_val
print "label_distance_val", label_distance_val
for i in xrange(video_id_equal_val.shape[0]):
if video_id_equal_val[i] > 0:
print "%d discrepancies in %s" % (int(video_id_equal_val[i]), all_patterns[i])
seconds_per_batch = time.time() - batch_start_time
example_per_second = video_id_equal_val.shape[0] / seconds_per_batch
examples_processed += video_id_equal_val.shape[0]
logging.info("examples_processed: %d", examples_processed)
except tf.errors.OutOfRangeError as e:
logging.info(
"Done with batched inference. Now calculating global performance "
"metrics.")
except Exception as e: # pylint: disable=broad-except
logging.info("Unexpected exception: " + str(e))
coord.request_stop(e)
coord.request_stop()
coord.join(threads, stop_grace_period_secs=10)
return global_step_val
def check_video_id():
tf.set_random_seed(0) # for reproducibility
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
# prepare a reader for each single model prediction result
all_readers = []
all_patterns = FLAGS.eval_data_patterns
all_patterns = map(lambda x: x.strip(), all_patterns.strip().strip(",").split(","))
for i in xrange(len(all_patterns)):
reader = readers.EnsembleReader(
feature_names=feature_names, feature_sizes=feature_sizes)
all_readers.append(reader)
input_reader = None
input_data_pattern = None
if FLAGS.input_data_pattern is not None:
input_reader = readers.EnsembleReader(
feature_names=["mean_rgb","mean_audio"], feature_sizes=[1024,128])
input_data_pattern = FLAGS.input_data_pattern
if FLAGS.eval_data_patterns is "":
raise IOError("'eval_data_patterns' was not specified. " +
"Nothing to evaluate.")
build_graph(
all_readers=all_readers,
input_reader=input_reader,
input_data_pattern=input_data_pattern,
all_eval_data_patterns=all_patterns,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
video_id_equal = tf.get_collection("video_id_equal")[0]
model_input = tf.get_collection("model_input")[0]
input_distance = tf.get_collection("input_distance")[0]
label_distance = tf.get_collection("label_distance")[0]
check_loop(model_input, video_id_equal, input_distance, label_distance, all_patterns)
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
check_video_id()
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-ensemble/check_video_id.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for evaluating Tensorflow models on the YouTube-8M dataset."""
import time
import eval_util
import losses
import ensemble_level_models
import readers
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import utils
FLAGS = flags.FLAGS
if __name__ == "__main__":
# Dataset flags.
flags.DEFINE_string("model_checkpoint_path", "",
"The file to load the model files from. ")
flags.DEFINE_string("train_dir", "/tmp/yt8m/",
"The directory to write the result in. ")
flags.DEFINE_string(
"eval_data_patterns", "",
"File globs defining the evaluation dataset in tensorflow.SequenceExample format.")
flags.DEFINE_string(
"input_data_pattern", None,
"File globs for original model input.")
flags.DEFINE_string("feature_names", "predictions", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "4716", "Length of the feature vectors.")
# Model flags.
flags.DEFINE_string(
"model", "LinearRegressionModel",
"Which architecture to use for the model.")
flags.DEFINE_integer("batch_size", 1024,
"How many examples to process per batch.")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Loss computed on validation data")
# Other flags.
flags.DEFINE_boolean("run_once", True, "Whether to run eval only once.")
flags.DEFINE_boolean("echo_gap", False, "Whether to echo GAP at the end.")
flags.DEFINE_integer("top_k", 20, "How many predictions to output per video.")
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def get_input_evaluation_tensors(reader,
data_pattern,
batch_size=256):
logging.info("Using batch size of " + str(batch_size) + " for evaluation.")
with tf.name_scope("eval_input"):
files = gfile.Glob(data_pattern)
if not files:
print data_pattern, files
raise IOError("Unable to find the evaluation files.")
logging.info("number of evaluation files: " + str(len(files)))
files.sort()
filename_queue = tf.train.string_input_producer(
files, shuffle=False, num_epochs=1)
eval_data = reader.prepare_reader(filename_queue)
return tf.train.batch(
eval_data,
batch_size=batch_size,
capacity=3 * batch_size,
allow_smaller_final_batch=True,
enqueue_many=True)
def build_graph(all_readers,
input_reader,
input_data_pattern,
all_eval_data_patterns,
batch_size=256):
original_video_id, original_input, unused_labels_batch, unused_num_frames = (
get_input_evaluation_tensors(
input_reader,
input_data_pattern,
batch_size=batch_size))
video_id_equal_tensors = []
model_input_tensor = None
input_distance_tensors = []
for reader, data_pattern in zip(all_readers, all_eval_data_patterns):
video_id, model_input_raw, labels_batch, unused_num_frames = (
get_input_evaluation_tensors(
reader,
data_pattern,
batch_size=batch_size))
video_id_equal_tensors.append(tf.reduce_sum(tf.cast(tf.not_equal(original_video_id, video_id), dtype=tf.float32)))
if model_input_tensor is None:
model_input_tensor = model_input_raw
input_distance_tensors.append(tf.reduce_mean(tf.reduce_sum(tf.square(model_input_tensor - model_input_raw), axis=1)))
video_id_equal_tensor = tf.stack(video_id_equal_tensors)
input_distance_tensor = tf.stack(input_distance_tensors)
tf.add_to_collection("video_id_equal", video_id_equal_tensor)
tf.add_to_collection("input_distance", input_distance_tensor)
def check_loop(video_id_equal, input_distance, all_patterns):
with tf.Session() as sess:
sess.run([tf.local_variables_initializer()])
# Start the queue runners.
coord = tf.train.Coordinator()
try:
threads = []
for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
threads.extend(qr.create_threads(
sess, coord=coord, daemon=True,
start=True))
examples_processed = 0
while not coord.should_stop():
batch_start_time = time.time()
video_id_equal_val, input_distance_val = sess.run([video_id_equal, input_distance])
print input_distance_val
for i in xrange(video_id_equal_val.shape[0]):
if video_id_equal_val[i] > 0:
print "%d discrepancies in %s" % (int(video_id_equal_val[i]), all_patterns[i])
seconds_per_batch = time.time() - batch_start_time
example_per_second = video_id_equal_val.shape[0] / seconds_per_batch
examples_processed += video_id_equal_val.shape[0]
logging.info("examples_processed: %d", examples_processed)
except tf.errors.OutOfRangeError as e:
logging.info(
"Done with batched inference. Now calculating global performance "
"metrics.")
except Exception as e: # pylint: disable=broad-except
logging.info("Unexpected exception: " + str(e))
coord.request_stop(e)
coord.request_stop()
coord.join(threads, stop_grace_period_secs=10)
return global_step_val
def check_video_id():
tf.set_random_seed(0) # for reproducibility
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
# prepare a reader for each single model prediction result
all_readers = []
all_patterns = FLAGS.eval_data_patterns
all_patterns = map(lambda x: x.strip(), all_patterns.strip().strip(",").split(","))
for i in xrange(len(all_patterns)):
reader = readers.EnsembleReader(
feature_names=feature_names, feature_sizes=feature_sizes)
all_readers.append(reader)
input_reader = None
input_data_pattern = None
if FLAGS.input_data_pattern is not None:
input_reader = readers.EnsembleReader(
feature_names=["mean_rgb","mean_audio"], feature_sizes=[1024,128])
input_data_pattern = FLAGS.input_data_pattern
if FLAGS.eval_data_patterns is "":
raise IOError("'eval_data_patterns' was not specified. " +
"Nothing to evaluate.")
build_graph(
all_readers=all_readers,
input_reader=input_reader,
input_data_pattern=input_data_pattern,
all_eval_data_patterns=all_patterns,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
video_id_equal = tf.get_collection("video_id_equal")[0]
input_distance = tf.get_collection("input_distance")[0]
check_loop(video_id_equal, input_distance, all_patterns)
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
check_video_id()
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-ensemble/check_video_id_match.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for evaluating Tensorflow models on the YouTube-8M dataset."""
import time
import eval_util
import losses
import ensemble_level_models
import readers
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import utils
FLAGS = flags.FLAGS
if __name__ == "__main__":
# Dataset flags.
flags.DEFINE_string("model_checkpoint_path", "",
"The file to load the model files from. ")
flags.DEFINE_string("train_dir", "/tmp/yt8m/",
"The directory to write the result in. ")
flags.DEFINE_string(
"eval_data_patterns", "",
"File globs defining the evaluation dataset in tensorflow.SequenceExample format.")
flags.DEFINE_string(
"input_data_pattern", None,
"File globs for original model input.")
flags.DEFINE_string("feature_names", "predictions", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "4716", "Length of the feature vectors.")
# Model flags.
flags.DEFINE_string(
"model", "LinearRegressionModel",
"Which architecture to use for the model.")
flags.DEFINE_integer("batch_size", 1024,
"How many examples to process per batch.")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Loss computed on validation data")
# Other flags.
flags.DEFINE_boolean("run_once", True, "Whether to run eval only once.")
flags.DEFINE_boolean("echo_gap", False, "Whether to echo GAP at the end.")
flags.DEFINE_integer("top_k", 20, "How many predictions to output per video.")
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def get_input_evaluation_tensors(reader,
data_pattern,
batch_size=256):
logging.info("Using batch size of " + str(batch_size) + " for evaluation.")
with tf.name_scope("eval_input"):
files = gfile.Glob(data_pattern)
if not files:
print data_pattern, files
raise IOError("Unable to find the evaluation files.")
logging.info("number of evaluation files: " + str(len(files)))
files.sort()
filename_queue = tf.train.string_input_producer(
files, shuffle=False, num_epochs=1)
eval_data = reader.prepare_reader(filename_queue)
return tf.train.batch(
eval_data,
batch_size=batch_size,
capacity=3 * FLAGS.batch_size,
allow_smaller_final_batch=True,
enqueue_many=True)
def build_graph(all_readers,
input_reader,
input_data_pattern,
all_eval_data_patterns,
batch_size=256):
original_video_id, original_input, unused_labels_batch, unused_num_frames = (
get_input_evaluation_tensors(
input_reader,
input_data_pattern,
batch_size=batch_size))
video_id_notequal_tensors = []
model_input_tensor = None
input_distance_tensors = []
for reader, data_pattern in zip(all_readers, all_eval_data_patterns):
video_id, model_input_raw, labels_batch, unused_num_frames = (
get_input_evaluation_tensors(
reader,
data_pattern,
batch_size=batch_size))
video_id_notequal_tensors.append(tf.reduce_sum(tf.cast(tf.not_equal(original_video_id, video_id), dtype=tf.float32)))
if model_input_tensor is None:
model_input_tensor = model_input_raw
input_distance_tensors.append(tf.reduce_mean(tf.reduce_sum(tf.square(model_input_tensor - model_input_raw), axis=1)))
video_id_mismatch_tensor = tf.stack(video_id_notequal_tensors)
input_distance_tensor = tf.stack(input_distance_tensors)
actual_batch_size = tf.shape(original_video_id)[0]
tf.add_to_collection("video_id_mismatch", video_id_mismatch_tensor)
tf.add_to_collection("input_distance", input_distance_tensor)
tf.add_to_collection("actual_batch_size", actual_batch_size)
def check_loop(video_id_mismatch, input_distance, actual_batch_size, all_patterns):
count_mismatch = [0] * len(all_patterns)
sum_distance = [0.0] * len(all_patterns)
with tf.Session() as sess:
sess.run([tf.local_variables_initializer()])
# Start the queue runners.
start_time = time.time()
coord = tf.train.Coordinator()
try:
threads = []
for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
threads.extend(qr.create_threads(
sess, coord=coord, daemon=True,
start=True))
examples_processed = 0
j = 0
while not coord.should_stop():
batch_start_time = time.time()
video_id_mismatch_val, input_distance_val, batch_size_val = sess.run([video_id_mismatch, input_distance, actual_batch_size])
for i in xrange(len(all_patterns)):
count_mismatch[i] += int(video_id_mismatch_val[i])
sum_distance[i] += float(input_distance_val[i])
print "mismatch", count_mismatch
seconds_per_batch = time.time() - batch_start_time
example_per_second = batch_size_val / seconds_per_batch
examples_processed += batch_size_val
logging.info("examples_processed: %d, time elapsed %f s", examples_processed, time.time()-start_time)
j += 1
if j % 100 == 0:
print "---------------- SUMMARY -------------------"
print "COUNT \t DIST \t PATTERNS"
for i in xrange(len(all_patterns)):
print count_mismatch[i], "\t", sum_distance[i], "\t", all_patterns[i]
except tf.errors.OutOfRangeError as e:
logging.info(
"Done with batched inference. Now calculating global performance "
"metrics.")
except Exception as e: # pylint: disable=broad-except
logging.info("Unexpected exception: " + str(e))
coord.request_stop(e)
coord.request_stop()
coord.join(threads, stop_grace_period_secs=10)
print "---------------- FINAL SUMMARY -------------------"
print "COUNT \t DIST \t PATTERNS"
for i in xrange(len(all_patterns)):
print count_mismatch[i], "\t", sum_distance[i], "\t", all_patterns[i]
def check_video_id():
tf.set_random_seed(0) # for reproducibility
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
# prepare a reader for each single model prediction result
all_readers = []
all_patterns = FLAGS.eval_data_patterns
all_patterns = map(lambda x: x.strip(), all_patterns.strip().strip(",").split(","))
for i in xrange(len(all_patterns)):
reader = readers.EnsembleReader(
feature_names=feature_names, feature_sizes=feature_sizes)
all_readers.append(reader)
input_reader = None
input_data_pattern = None
if FLAGS.input_data_pattern is not None:
input_reader = readers.EnsembleReader(
feature_names=["mean_rgb","mean_audio"], feature_sizes=[1024,128])
# input_reader = readers.EnsembleReader(
# feature_names=["input"], feature_sizes=[1024+128])
input_data_pattern = FLAGS.input_data_pattern
if FLAGS.eval_data_patterns is "":
raise IOError("'eval_data_patterns' was not specified. " +
"Nothing to evaluate.")
build_graph(
all_readers=all_readers,
input_reader=input_reader,
input_data_pattern=input_data_pattern,
all_eval_data_patterns=all_patterns,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
video_id_mismatch = tf.get_collection("video_id_mismatch")[0]
input_distance = tf.get_collection("input_distance")[0]
actual_batch_size = tf.get_collection("actual_batch_size")[0]
check_loop(video_id_mismatch, input_distance, actual_batch_size, all_patterns)
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
check_video_id()
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-ensemble/cloudml-gpu-distributed.yaml
================================================
trainingInput:
runtimeVersion: "1.0"
scaleTier: CUSTOM
masterType: standard_gpu
workerCount: 2
workerType: standard_gpu
parameterServerCount: 2
parameterServerType: standard
================================================
FILE: youtube-8m-ensemble/cloudml-gpu.yaml
================================================
trainingInput:
scaleTier: CUSTOM
# standard_gpu provides 1 GPU. Change to complex_model_m_gpu for 4 GPUs
masterType: standard_gpu
runtimeVersion: "1.0"
================================================
FILE: youtube-8m-ensemble/data_augmentation.py
================================================
from tensorflow import flags
flags.DEFINE_string("data_augmenter", "DefaultAugmenter",
"how to preprocess feature, defaults to identical, which means no transform")
flags.DEFINE_float("input_noise_level", 0.2,
"the standard deviation of normal noise added to input")
from all_data_augmentation import *
================================================
FILE: youtube-8m-ensemble/ensemble_command.example
================================================
# bash [script] [model_name] [config]
# use a different model name each time, make it understandable
# create a new conf file, add it to git repository
bash ensemble_scripts/train-mean_model.sh ensemble_mean_model_10 ensemble_scripts/ensemble_no3.conf
bash ensemble_scripts/eval-mean_model.sh ensemble_mean_model_10 ensemble_scripts/ensemble_no3.conf
bash ensemble_scripts/infer-mean_model.sh ensemble_mean_model_10 ensemble_scripts/ensemble_no3.conf
================================================
FILE: youtube-8m-ensemble/ensemble_level_models.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains model definitions."""
from tensorflow import flags
FLAGS = flags.FLAGS
flags.DEFINE_integer(
"moe_num_mixtures", 2,
"The number of mixtures (excluding the dummy 'expert') used for MoeModel.")
flags.DEFINE_integer(
"deep_chain_layers", 3,
"The number of layers used for DeepChainModel")
flags.DEFINE_integer(
"deep_chain_relu_cells", 200,
"The number of relu cells used for DeepChainModel")
flags.DEFINE_string(
"deep_chain_relu_type", "relu",
"The type of relu cells used for DeepChainModel (options are elu and relu)")
flags.DEFINE_bool(
"deep_chain_use_length", False,
"The number of relu cells used for DeepChainModel")
flags.DEFINE_integer(
"hidden_chain_layers", 4,
"The number of layers used for HiddenChainModel")
flags.DEFINE_integer(
"hidden_chain_relu_cells", 256,
"The number of relu cells used for HiddenChainModel")
flags.DEFINE_integer(
"attention_matrix_rank", 3,
"The rank of weight matrix used for AttentionMatrixModel.")
flags.DEFINE_integer(
"attention_relu_cells", 128,
"The rank of weight matrix used for AttentionMatrixModel.")
import sys
from os.path import dirname
if dirname(__file__) not in sys.path:
sys.path.append(dirname(__file__))
from all_ensemble_models import *
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/.vimrc
================================================
set tabstop=2
set shiftwidth=2
set expandtab
set autoindent
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/after_submission_no1.conf
================================================
video_relabel_combine_chain
video_very_deep_combine_chain
lstmmemory_cell1024_layer2_moe8
lstmmemory_cell2048_layer2_moe4
lstmparallelfinaloutput1024_moe8
lstmparallelmemory1024_moe8
lstmgate_cell1024_layer1_moe8
lstmglu2_cell1024_layer1_moe8
lstmbiglu_cell1024_layer1_moe8
biunilstm1024_moe4
lstm_random_mean_moe8
lstm_shortlayers_moe8
framehop_lstm
cnnlstmmemory1024_moe8
attentionlstm_moe4
lstmattlstm1024_moe8
lstm_attention8_max
lstm2_attention8_max
lstm_positional_attention8max
cnn_deep_combine_chain
deep_cnn_deep_combine_chain
lstm_cnn_deep_combine_chain
multilstmmemory1024_moe4_deep_chain
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
multires_lstm_deep_combine_chain
lstm_gate_multiscale4_moe4
lstm_multiscale4_moe4
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/after_submission_no2.conf
================================================
video_relabel_combine_chain
video_very_deep_combine_chain
lstmmemory_cell1024_layer2_moe8
lstmmemory_cell2048_layer2_moe4
lstmparallelfinaloutput1024_moe8
lstmparallelmemory1024_moe8
lstmgate_cell1024_layer1_moe8
lstmglu2_cell1024_layer1_moe8
lstmbiglu_cell1024_layer1_moe8
biunilstm1024_moe4
lstm_random_mean_moe8
lstm_shortlayers_moe8
framehop_lstm
cnnlstmmemory1024_moe8
attentionlstm_moe4
lstmattlstm1024_moe8
lstm_attention8_max
lstm2_attention8_max
lstm_positional_attention8max
cnn_deep_combine_chain
deep_cnn_deep_combine_chain
lstm_cnn_deep_combine_chain
multilstmmemory1024_moe4_deep_chain
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
multires_lstm_deep_combine_chain
lstm_gate_multiscale4_moe4
lstm_multiscale4_moe4
video_dcc_bagging
video_cc_structure_bagging/ensemble_matrix_model
video_dcc_boosting_discardhopeless/ensemble_matrix_model
video_dcc_boosting/ensemble_matrix_model
video_dcc_boosting_weightclip/ensemble_matrix_model
cnn_deep_combine_chain_bagging/ensemble_matrix_model
cnn_deep_combine_chain_boosting/ensemble_attention_matrix_model
lstmattention8max_bagging/ensemble_matrix_model
lstmattention8max_boosting/ensemble_attention_matrix_model
lstmparalleloutput_bagging/ensemble_mean_model
lstmparalleloutput_boosting_weightclip/ensemble_matrix_model
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/after_submission_no3.conf
================================================
video_relabel_combine_chain
video_very_deep_combine_chain
lstmmemory_cell1024_layer2_moe8
lstmmemory_cell2048_layer2_moe4
lstmparallelfinaloutput1024_moe8
lstmparallelmemory1024_moe8
lstmgate_cell1024_layer1_moe8
lstmglu2_cell1024_layer1_moe8
lstmbiglu_cell1024_layer1_moe8
biunilstm1024_moe4
lstm_random_mean_moe8
lstm_shortlayers_moe8
framehop_lstm
cnnlstmmemory1024_moe8
attentionlstm_moe4
lstmattlstm1024_moe8
lstm_attention8_max
lstm2_attention8_max
lstm_positional_attention8max
cnn_deep_combine_chain
deep_cnn_deep_combine_chain
lstm_cnn_deep_combine_chain
multilstmmemory1024_moe4_deep_chain
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
multires_lstm_deep_combine_chain
lstm_gate_multiscale4_moe4
lstm_multiscale4_moe4
video_dcc_bagging
video_cc_structure_bagging/ensemble_matrix_model
video_dcc_boosting_discardhopeless/ensemble_matrix_model
video_dcc_boosting/ensemble_matrix_model
video_dcc_boosting_weightclip/ensemble_matrix_model
cnn_deep_combine_chain_bagging/ensemble_matrix_model
cnn_deep_combine_chain_boosting/ensemble_attention_matrix_model
lstmattention8max_bagging/ensemble_matrix_model
lstmattention8max_boosting/ensemble_attention_matrix_model
lstmparalleloutput_bagging/ensemble_mean_model
lstmparalleloutput_boosting_weightclip/ensemble_matrix_model
distillation_video_dcc_noise
distillation_cnn_dcc_boosting/sub_model_1
distillation_lstmcnn_dcc_boosting/sub_model_1
distillation_lstmparalleloutput_boosting/sub_model_1
distillation_multilstm_dcc_boosting/sub_model_1
distillation_multiscale_cnnlstm_boosting/sub_model_1
distillation_video_dcc_bagging/ensemble_matrix_model
distillation_video_dcc_boosting/ensemble_matrix_model
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/after_submission_no4.conf
================================================
video_relabel_combine_chain
video_very_deep_combine_chain
lstmmemory_cell1024_layer2_moe8
lstmmemory_cell2048_layer2_moe4
lstmparallelfinaloutput1024_moe8
lstmparallelmemory1024_moe8
lstmgate_cell1024_layer1_moe8
lstmglu2_cell1024_layer1_moe8
lstmbiglu_cell1024_layer1_moe8
biunilstm1024_moe4
lstm_random_mean_moe8
lstm_shortlayers_moe8
framehop_lstm
cnnlstmmemory1024_moe8
attentionlstm_moe4
lstmattlstm1024_moe8
lstm_attention8_max
lstm2_attention8_max
lstm_positional_attention8max
cnn_deep_combine_chain
deep_cnn_deep_combine_chain
lstm_cnn_deep_combine_chain
multilstmmemory1024_moe4_deep_chain
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
multires_lstm_deep_combine_chain
lstm_gate_multiscale4_moe4
lstm_multiscale4_moe4
video_dcc_bagging
video_cc_structure_bagging/ensemble_matrix_model
video_dcc_boosting_discardhopeless/ensemble_matrix_model
video_dcc_boosting/ensemble_matrix_model
video_dcc_boosting_weightclip/ensemble_matrix_model
cnn_deep_combine_chain_bagging/ensemble_matrix_model
cnn_deep_combine_chain_boosting/ensemble_attention_matrix_model
lstmattention8max_bagging/ensemble_matrix_model
lstmattention8max_boosting/ensemble_attention_matrix_model
lstmparalleloutput_bagging/ensemble_mean_model
lstmparalleloutput_boosting_weightclip/ensemble_matrix_model
distillation_video_dcc_noise
distillation_cnn_dcc_boosting/sub_model_1
distillation_lstmcnn_dcc_boosting/sub_model_1
distillation_lstmparalleloutput_boosting/sub_model_1
distillation_multilstm_dcc_boosting/sub_model_1
distillation_multiscale_cnnlstm_boosting/sub_model_1
distillation_video_dcc_bagging/ensemble_matrix_model
distillation_video_dcc_boosting/ensemble_matrix_model
distillchain_video_dcc
distillchain_video_norm_moe8
distillchain_lstm_gate_moe8
distillchain_cnn_dcc
distillchain_cnndcc_layer2moe4_ensemble
distillchain_lstm_attention8max
distillchain_lstm_moe8
distillchain_lstm_multiscale_layer2_moe8
distillchain_lstm_multiscale_layer4_moe8
distillchain_lstmcnn_dcc
distillsplit_lstm_gate_moe8
distillchain_v2_video_dcc
distillchain_v2_lstmattention8max
distillchain_v2_lstmcnn_dcc
distillchain_v2_lstm_gate_moe8
distillchain_v2_lstm_glu2_moe8
distillchain_v2_multilstm_dcc
distillchain_v2_multiscale_cnnlstm
distillchain_v2_lstmparalleloutput
distillchain_v2_boost_lstmparalleloutput
videochain/distillchain_v2_video_dcc_1
videochain/distillchain_v2_video_dcc_2
videochain/distillchain_v2_video_dcc_3
videochain/distillchain_v2_video_dcc_4
videochain/distillchain_v2_video_dcc_5
hybridchain/distillchain_v2_video_dcc
hybridchain/distillchain_v2_cnn_dcc
hybridchain/distillchain_v2_lstmparalleloutput
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/auto-preensemble-deep_combine_chain_model.sh
================================================
model=$1
conf=$2
part=$3
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
test_path=/Youtube-8M/model_predictions/${part}
test_data_patterns=""
for d in $(cat $conf); do
test_data_patterns="${test_path}/${d}/*.tfrecord${test_data_patterns:+,$test_data_patterns}"
done
echo "$test_data_patterns"
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/${model}" \
--train_dir="../model/${model}" \
--input_data_patterns="$test_data_patterns" \
--model="DeepCombineChainModel" \
--deep_chain_relu_cells=128 \
--moe_num_mixtures=2 \
--batch_size=512 \
--file_size=4096
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/auto-preensemble-matrix_model.sh
================================================
model=$1
conf=$2
part=$3
postfix=$4
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
test_path=/Youtube-8M/model_predictions${postfix}/${part}
test_data_patterns=""
for d in $(cat $conf); do
test_data_patterns="${test_path}/${d}/*.tfrecord${test_data_patterns:+,$test_data_patterns}"
done
echo "$test_data_patterns"
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/${model}" \
--train_dir="../model/${model}" \
--input_data_patterns="$test_data_patterns" \
--model="MatrixRegressionModel" \
--batch_size=1024 \
--file_size=4096
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/check-video_id.sh
================================================
conf=$1
part=$2
validate_path=/Youtube-8M/model_predictions/${part}
validate_data_patterns=""
for d in $(cat $conf); do
validate_data_patterns="${validate_path}/${d}/*.tfrecord${validate_data_patterns:+,$validate_data_patterns}"
done
echo "$validate_data_patterns"
input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord"
CUDA_VISIBLE_DEVICES="" python check_video_id.py \
--input_data_pattern=$input_data_pattern \
--eval_data_patterns="$validate_data_patterns"
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/check-video_id_match.sh
================================================
conf=$1
part=$2
validate_path=/Youtube-8M/model_predictions_local/${part}
validate_data_patterns=""
for d in $(cat $conf); do
validate_data_patterns="${validate_path}/${d}/*.tfrecord${validate_data_patterns:+,$validate_data_patterns}"
done
echo "$validate_data_patterns"
input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord"
#input_data_pattern="/Youtube-8M/model_predictions/${part}/model_input/*.tfrecord"
CUDA_VISIBLE_DEVICES="" python check_video_id_match.py \
--input_data_pattern=$input_data_pattern \
--eval_data_patterns="$validate_data_patterns"
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/combine-tfrecords-frame-v2.sh
================================================
#!/bin/bash
file_num_mod=$1
data_path=/Youtube-8M/model_predictions_x32/train
input_data_pattern="/Youtube-8M/data/frame/train/*.tfrecord"
prediction_data_pattern="${data_path}/distillation/ensemble_v2_matrix_model/prediction*.tfrecord"
CUDA_VISIBLE_DEVICES="" python inference-combine-tfrecords-frame.py \
--output_dir="/Youtube-8M/distillation_v2/frame/train" \
--input_data_pattern="$input_data_pattern" \
--prediction_data_pattern="$prediction_data_pattern" \
--file_num_mod=$file_num_mod \
--batch_size=128 \
--file_size=1024
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/combine-tfrecords-frame.sh
================================================
#!/bin/bash
data_path=/Youtube-8M/model_predictions/train
input_data_pattern="/Youtube-8M/data/frame/train/train*.tfrecord"
prediction_data_pattern="${data_path}/distillation/ensemble_mean_model/prediction*.tfrecord"
CUDA_VISIBLE_DEVICES="" python inference-combine-tfrecords-frame.py \
--output_dir="/Youtube-8M/distillation/frame/train" \
--input_data_pattern="$input_data_pattern" \
--prediction_data_pattern="$prediction_data_pattern" \
--batch_size=128 \
--file_size=1024
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/combine-tfrecords-video-v2.sh
================================================
#!/bin/bash
data_path=/Youtube-8M/model_predictions_local/train
input_data_pattern="/Youtube-8M/data/video/train/*.tfrecord"
prediction_data_pattern="${data_path}/distillation/ensemble_v2_matrix_model/prediction*.tfrecord"
CUDA_VISIBLE_DEVICES="" python inference-combine-tfrecords-video.py \
--output_dir="/Youtube-8M/distillation_v2/video/train" \
--input_data_pattern="$input_data_pattern" \
--prediction_data_pattern="$prediction_data_pattern" \
--batch_size=1024 \
--file_size=4096
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/combine-tfrecords-video.sh
================================================
#!/bin/bash
data_path=/Youtube-8M/model_predictions/train
input_data_pattern="/Youtube-8M/data/video/train/train*.tfrecord"
prediction_data_pattern="${data_path}/distillation/ensemble_mean_model/prediction*.tfrecord"
CUDA_VISIBLE_DEVICES="" python inference-combine-tfrecords-video.py \
--output_dir="/Youtube-8M/distillation/video/train" \
--input_data_pattern="$input_data_pattern" \
--prediction_data_pattern="$prediction_data_pattern" \
--batch_size=1024 \
--file_size=4096
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/ensemble_no1.conf
================================================
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
lstmmemory_cell2048_layer2_moe4
video_very_deep_combine_chain
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/ensemble_no10.conf
================================================
attentionlstm_moe4
biunilstm1024_moe4
cnn_deep_combine_chain
cnnlstmmemory1024_moe8
deep_cnn_deep_combine_chain
framehop_lstm
lstm2_attention8_max
lstm_attention8_max
lstmattlstm1024_moe8
lstm_cnn_deep_combine_chain
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
lstmmemory_cell1024_layer2_moe8
lstmmemory_cell2048_layer2_moe4
lstmparallelfinaloutput1024_moe8
lstmparallelmemory1024_moe8
lstm_positional_attention8max
lstm_random_mean_moe8
lstm_shortlayers_moe8
multilstmmemory1024_moe4_deep_chain
multires_lstm_deep_combine_chain
video_relabel_combine_chain
video_very_deep_combine_chain
distillation_video_dcc_noise
video_cc_structure_bagging/ensemble_matrix_model
video_dcc_bagging
video_dcc_boosting/ensemble_matrix_model
video_dcc_boosting_discardhopeless/ensemble_matrix_model
video_dcc_boosting_weightclip/ensemble_matrix_model
cnn_deep_combine_chain_bagging/ensemble_matrix_model
cnn_deep_combine_chain_boosting/ensemble_matrix_model
lstmattention8max_bagging/ensemble_matrix_model
lstmattention8max_boosting/ensemble4_matrix_model
lstmparalleloutput_bagging/ensemble_mean_model
lstmparalleloutput_boosting_weightclip/ensemble_matrix_model
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/ensemble_no11.conf
================================================
attentionlstm_moe4
biunilstm1024_moe4
cnn_deep_combine_chain
cnn_deep_combine_chain_bagging/ensemble_matrix_model
cnn_deep_combine_chain_boosting/ensemble_matrix_model
cnnlstmmemory1024_moe8
deep_cnn_deep_combine_chain
distillation_video_dcc_bagging/ensemble_matrix_model
distillation_video_dcc_boosting/ensemble_matrix_model
distillation_video_dcc_noise
framehop_lstm
lstm2_attention8_max
lstm_attention8_max
lstmattention8max_bagging/ensemble_matrix_model
lstmattention8max_boosting/ensemble4_matrix_model
lstmattlstm1024_moe8
lstm_cnn_deep_combine_chain
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
lstmmemory_cell1024_layer2_moe8
lstmmemory_cell2048_layer2_moe4
lstmparallelfinaloutput1024_moe8
lstmparallelmemory1024_moe8
lstmparalleloutput_bagging/ensemble_mean_model
lstmparalleloutput_boosting_weightclip/ensemble_matrix_model
lstm_positional_attention8max
lstm_random_mean_moe8
lstm_shortlayers_moe8
multilstmmemory1024_moe4_deep_chain
multires_lstm_deep_combine_chain
video_cc_structure_bagging/ensemble_matrix_model
video_dcc_bagging
video_dcc_boosting_discardhopeless/ensemble_matrix_model
video_dcc_boosting/ensemble_matrix_model
video_dcc_boosting_weightclip/ensemble_matrix_model
video_relabel_combine_chain
video_very_deep_combine_chain
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/ensemble_no12.conf
================================================
attentionlstm_moe4
biunilstm1024_moe4
cnn_deep_combine_chain
cnn_deep_combine_chain_bagging/ensemble_matrix_model
cnn_deep_combine_chain_boosting/ensemble_matrix_model
cnnlstmmemory1024_moe8
deep_cnn_deep_combine_chain
distillation_video_dcc_bagging/ensemble_matrix_model
distillation_video_dcc_boosting/ensemble_matrix_model
distillation_video_dcc_noise
framehop_lstm
lstm2_attention8_max
lstm_attention8_max
lstmattention8max_bagging/ensemble_matrix_model
lstmattention8max_boosting/ensemble4_matrix_model
lstmattlstm1024_moe8
lstm_cnn_deep_combine_chain
lstmgate_cell1024_layer1_moe8
lstmglu2_cell1024_layer1_moe8
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
lstmmemory_cell1024_layer2_moe8
lstmmemory_cell2048_layer2_moe4
lstm_multiscale4_moe4
lstmparallelfinaloutput1024_moe8
lstmparallelmemory1024_moe8
lstmparalleloutput_bagging/ensemble_mean_model
lstmparalleloutput_boosting_weightclip/ensemble_matrix_model
lstm_positional_attention8max
lstm_random_mean_moe8
lstm_shortlayers_moe8
multilstmmemory1024_moe4_deep_chain
multires_lstm_deep_combine_chain
video_cc_structure_bagging/ensemble_matrix_model
video_dcc_bagging
video_dcc_boosting_discardhopeless/ensemble_matrix_model
video_dcc_boosting/ensemble_matrix_model
video_dcc_boosting_weightclip/ensemble_matrix_model
video_relabel_combine_chain
video_very_deep_combine_chain
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/ensemble_no13.conf
================================================
attentionlstm_moe4
biunilstm1024_moe4
cnn_deep_combine_chain
cnn_deep_combine_chain_bagging/ensemble_matrix_model
cnn_deep_combine_chain_boosting/ensemble_matrix_model
cnn_deep_combine_chain_boosting/ensemble_attention_matrix_model
cnnlstmmemory1024_moe8
deep_cnn_deep_combine_chain
distillation_video_dcc_bagging/ensemble_matrix_model
distillation_video_dcc_boosting/ensemble_matrix_model
distillation_video_dcc_noise
framehop_lstm
lstm2_attention8_max
lstm_attention8_max
lstmattention8max_bagging/ensemble_matrix_model
lstmattention8max_boosting/ensemble4_matrix_model
lstmattention8max_boosting/sub_model_1
lstmattention8max_boosting/sub_model_2
lstmattention8max_boosting/sub_model_3
lstmattention8max_boosting/sub_model_4
lstmattention8max_boosting/sub_model_5
lstmattention8max_boosting/sub_model_6
lstmattention8max_boosting/sub_model_7
lstmattlstm1024_moe8
lstm_cnn_deep_combine_chain
lstmgate_cell1024_layer1_moe8
lstmglu2_cell1024_layer1_moe8
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
lstmmemory_cell1024_layer2_moe8
lstmmemory_cell2048_layer2_moe4
lstm_multiscale4_moe4
lstmparallelfinaloutput1024_moe8
lstmparallelmemory1024_moe8
lstmparalleloutput_bagging/ensemble_mean_model
lstmparalleloutput_boosting_weightclip/ensemble_matrix_model
lstm_positional_attention8max
lstm_random_mean_moe8
lstm_shortlayers_moe8
multilstmmemory1024_moe4_deep_chain
multires_lstm_deep_combine_chain
video_cc_structure_bagging/ensemble_matrix_model
video_cc_structure_bagging/video_knowledge_combine_chain
video_cc_structure_bagging/video_notzero_combine_chain
video_cc_structure_bagging/video_softmax_combine_chain
video_cc_structure_bagging/video_weight_combine_chain
video_dcc_bagging
video_dcc_boosting_discardhopeless/ensemble_matrix_model
video_dcc_boosting_discardhopeless/sub_model_1
video_dcc_boosting_discardhopeless/sub_model_2
video_dcc_boosting_discardhopeless/sub_model_3
video_dcc_boosting_discardhopeless/sub_model_4
video_dcc_boosting_discardhopeless/sub_model_5
video_dcc_boosting_discardhopeless/sub_model_6
video_dcc_boosting_discardhopeless/sub_model_7
video_dcc_boosting_discardhopeless/sub_model_8
video_dcc_boosting/ensemble_matrix_model
video_dcc_boosting_weightclip/ensemble_matrix_model
video_relabel_combine_chain
video_very_deep_combine_chain
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/ensemble_no14.conf
================================================
attentionlstm_moe4
biunilstm1024_moe4
cnn_deep_combine_chain
cnn_deep_combine_chain_bagging/ensemble_matrix_model
cnn_deep_combine_chain_boosting/ensemble_attention_matrix_model
cnnlstmmemory1024_moe8
deep_cnn_deep_combine_chain
distillation_video_dcc_bagging/ensemble_matrix_model
distillation_video_dcc_boosting/ensemble_matrix_model
distillation_video_dcc_noise
framehop_lstm
lstm2_attention8_max
lstm_attention8_max
lstmattention8max_bagging/ensemble_matrix_model
lstmattention8max_boosting/ensemble_attention_matrix_model
lstmattlstm1024_moe8
lstm_cnn_deep_combine_chain
lstmgate_cell1024_layer1_moe8
lstmglu2_cell1024_layer1_moe8
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
lstmmemory_cell1024_layer2_moe8
lstmmemory_cell2048_layer2_moe4
lstm_multiscale4_moe4
lstmparallelfinaloutput1024_moe8
lstmparallelmemory1024_moe8
lstmparalleloutput_bagging/ensemble_mean_model
lstmparalleloutput_boosting_weightclip/ensemble_matrix_model
lstm_positional_attention8max
lstm_random_mean_moe8
lstm_shortlayers_moe8
multilstmmemory1024_moe4_deep_chain
multires_lstm_deep_combine_chain
video_cc_structure_bagging/ensemble_matrix_model
video_dcc_bagging
video_dcc_boosting_discardhopeless/ensemble_matrix_model
video_dcc_boosting/ensemble_matrix_model
video_dcc_boosting_weightclip/ensemble_matrix_model
video_relabel_combine_chain
video_very_deep_combine_chain
distillation_cnn_dcc_boosting/sub_model_1
distillation_lstmcnn_dcc_boosting/sub_model_1
distillation_lstmparalleloutput_boosting/sub_model_1
distillation_multilstm_dcc_boosting/sub_model_1
distillation_multiscale_cnnlstm_boosting/sub_model_1
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/ensemble_no15.conf
================================================
attentionlstm_moe4
biunilstm1024_moe4
cnn_deep_combine_chain
cnn_deep_combine_chain_bagging/ensemble_matrix_model
cnn_deep_combine_chain_boosting/ensemble_attention_matrix_model
cnnlstmmemory1024_moe8
deep_cnn_deep_combine_chain
distillation_video_dcc_bagging/ensemble_matrix_model
distillation_video_dcc_boosting/ensemble_matrix_model
distillation_video_dcc_noise
framehop_lstm
lstm2_attention8_max
lstm_attention8_max
lstmattention8max_bagging/ensemble_matrix_model
lstmattention8max_boosting/ensemble_attention_matrix_model
lstmattlstm1024_moe8
lstm_cnn_deep_combine_chain
lstmgate_cell1024_layer1_moe8
lstmglu2_cell1024_layer1_moe8
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
lstmmemory_cell1024_layer2_moe8
lstmmemory_cell2048_layer2_moe4
lstm_multiscale4_moe4
lstmparallelfinaloutput1024_moe8
lstmparallelmemory1024_moe8
lstmparalleloutput_bagging/ensemble_mean_model
lstmparalleloutput_boosting_weightclip/ensemble_matrix_model
lstm_positional_attention8max
lstm_random_mean_moe8
lstm_shortlayers_moe8
multilstmmemory1024_moe4_deep_chain
multires_lstm_deep_combine_chain
video_cc_structure_bagging/ensemble_matrix_model
video_dcc_bagging
video_dcc_boosting_discardhopeless/ensemble_matrix_model
video_dcc_boosting/ensemble_matrix_model
video_dcc_boosting_weightclip/ensemble_matrix_model
video_relabel_combine_chain
video_very_deep_combine_chain
distillation_cnn_dcc_boosting/sub_model_1
distillation_lstmcnn_dcc_boosting/sub_model_1
distillation_lstmparalleloutput_boosting/sub_model_1
distillation_multilstm_dcc_boosting/sub_model_1
distillation_multiscale_cnnlstm_boosting/sub_model_1
video_logistic_model
video_moe16_model
dbof_model
frame_seg_model
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/ensemble_no16.conf
================================================
attentionlstm_moe4
biunilstm1024_moe4
cnn_deep_combine_chain
cnn_deep_combine_chain_bagging/ensemble_matrix_model
cnn_deep_combine_chain_boosting/ensemble_attention_matrix_model
cnnlstmmemory1024_moe8
deep_cnn_deep_combine_chain
distillation_video_dcc_bagging/ensemble_matrix_model
distillation_video_dcc_boosting/ensemble_matrix_model
distillation_video_dcc_noise
framehop_lstm
lstm2_attention8_max
lstm_attention8_max
lstmattention8max_bagging/ensemble_matrix_model
lstmattention8max_boosting/ensemble_attention_matrix_model
lstmattlstm1024_moe8
lstm_cnn_deep_combine_chain
lstmgate_cell1024_layer1_moe8
lstmglu2_cell1024_layer1_moe8
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
lstmmemory_cell1024_layer2_moe8
lstmmemory_cell2048_layer2_moe4
lstm_multiscale4_moe4
lstmparallelfinaloutput1024_moe8
lstmparallelmemory1024_moe8
lstmparalleloutput_bagging/ensemble_mean_model
lstmparalleloutput_boosting_weightclip/ensemble_matrix_model
lstm_positional_attention8max
lstm_random_mean_moe8
lstm_shortlayers_moe8
multilstmmemory1024_moe4_deep_chain
multires_lstm_deep_combine_chain
video_cc_structure_bagging/ensemble_matrix_model
video_dcc_bagging
video_dcc_boosting_discardhopeless/ensemble_matrix_model
video_dcc_boosting/ensemble_matrix_model
video_dcc_boosting_weightclip/ensemble_matrix_model
video_relabel_combine_chain
video_very_deep_combine_chain
distillation_cnn_dcc_boosting/sub_model_1
distillation_lstmcnn_dcc_boosting/sub_model_1
distillation_lstmparalleloutput_boosting/sub_model_1
distillation_multilstm_dcc_boosting/sub_model_1
distillation_multiscale_cnnlstm_boosting/sub_model_1
distillchain_lstm_gate_moe8
distillchain_video_dcc
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/ensemble_no17.conf
================================================
cnn_deep_combine_chain
cnn_deep_combine_chain_bagging/ensemble_matrix_model
distillation_cnn_dcc_boosting/sub_model_1
distillation_lstmcnn_dcc_boosting/sub_model_1
distillation_multilstm_dcc_boosting/sub_model_1
framehop_lstm
lstm_attention8_max
lstm_cnn_deep_combine_chain
lstm_positional_attention8max
lstmattention8max_bagging/ensemble_matrix_model
lstmattention8max_boosting/ensemble_attention_matrix_model
lstmbiglu_cell1024_layer1_moe8
lstmglu2_cell1024_layer1_moe8
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
lstmparallelfinaloutput1024_moe8
lstmparallelmemory1024_moe8
multilstmmemory1024_moe4_deep_chain
video_dcc_boosting/ensemble_matrix_model
video_dcc_boosting_discardhopeless/ensemble_matrix_model
video_dcc_boosting_weightclip/ensemble_matrix_model
distillchain_cnn_dcc
distillchain_cnndcc_layer2moe4
distillchain_lstm_attention8max
distillchain_lstm_gate_moe8
distillchain_lstm_moe8
distillchain_lstm_multiscale_layer2_moe8
distillchain_lstm_multiscale_layer4_moe8
distillchain_video_dcc
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/ensemble_no18.conf
================================================
attentionlstm_moe4
biunilstm1024_moe4
cnn_deep_combine_chain
cnn_deep_combine_chain_bagging/ensemble_matrix_model
cnn_deep_combine_chain_boosting/ensemble_attention_matrix_model
cnnlstmmemory1024_moe8
deep_cnn_deep_combine_chain
distillation_video_dcc_bagging/ensemble_matrix_model
distillation_video_dcc_boosting/ensemble_matrix_model
distillation_video_dcc_noise
framehop_lstm
lstm2_attention8_max
lstm_attention8_max
lstmattention8max_bagging/ensemble_matrix_model
lstmattention8max_boosting/ensemble_attention_matrix_model
lstmattlstm1024_moe8
lstm_cnn_deep_combine_chain
lstmgate_cell1024_layer1_moe8
lstmglu2_cell1024_layer1_moe8
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
lstmmemory_cell1024_layer2_moe8
lstmmemory_cell2048_layer2_moe4
lstm_multiscale4_moe4
lstmparallelfinaloutput1024_moe8
lstmparallelmemory1024_moe8
lstmparalleloutput_bagging/ensemble_mean_model
lstmparalleloutput_boosting_weightclip/ensemble_matrix_model
lstm_positional_attention8max
lstm_random_mean_moe8
lstm_shortlayers_moe8
multilstmmemory1024_moe4_deep_chain
multires_lstm_deep_combine_chain
video_cc_structure_bagging/ensemble_matrix_model
video_dcc_bagging
video_dcc_boosting_discardhopeless/ensemble_matrix_model
video_dcc_boosting/ensemble_matrix_model
video_dcc_boosting_weightclip/ensemble_matrix_model
video_relabel_combine_chain
video_very_deep_combine_chain
distillation_cnn_dcc_boosting/sub_model_1
distillation_lstmcnn_dcc_boosting/sub_model_1
distillation_lstmparalleloutput_boosting/sub_model_1
distillation_multilstm_dcc_boosting/sub_model_1
distillation_multiscale_cnnlstm_boosting/sub_model_1
distillchain_lstm_gate_moe8
distillchain_video_dcc
distillchain_cnn_dcc
distillchain_cnndcc_layer2moe4_ensemble
distillchain_lstm_attention8max
distillchain_lstm_moe8
distillchain_lstm_multiscale_layer2_moe8
distillchain_lstm_multiscale_layer4_moe8
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/ensemble_no19.conf
================================================
attentionlstm_moe4
biunilstm1024_moe4
cnn_deep_combine_chain
cnn_deep_combine_chain_bagging/ensemble_matrix_model
cnn_deep_combine_chain_boosting/ensemble_attention_matrix_model
cnnlstmmemory1024_moe8
deep_cnn_deep_combine_chain
distillation_video_dcc_bagging/ensemble_matrix_model
distillation_video_dcc_boosting/ensemble_matrix_model
distillation_video_dcc_noise
framehop_lstm
lstm2_attention8_max
lstm_attention8_max
lstmattention8max_bagging/ensemble_matrix_model
lstmattention8max_boosting/ensemble_attention_matrix_model
lstmattlstm1024_moe8
lstm_cnn_deep_combine_chain
lstmgate_cell1024_layer1_moe8
lstmglu2_cell1024_layer1_moe8
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
lstmmemory_cell1024_layer2_moe8
lstmmemory_cell2048_layer2_moe4
lstm_multiscale4_moe4
lstmparallelfinaloutput1024_moe8
lstmparallelmemory1024_moe8
lstmparalleloutput_bagging/ensemble_mean_model
lstmparalleloutput_boosting_weightclip/ensemble_matrix_model
lstm_positional_attention8max
lstm_random_mean_moe8
lstm_shortlayers_moe8
multilstmmemory1024_moe4_deep_chain
multires_lstm_deep_combine_chain
video_cc_structure_bagging/ensemble_matrix_model
video_dcc_bagging
video_dcc_boosting_discardhopeless/ensemble_matrix_model
video_dcc_boosting/ensemble_matrix_model
video_dcc_boosting_weightclip/ensemble_matrix_model
video_relabel_combine_chain
video_very_deep_combine_chain
distillation_cnn_dcc_boosting/sub_model_1
distillation_lstmcnn_dcc_boosting/sub_model_1
distillation_lstmparalleloutput_boosting/sub_model_1
distillation_multilstm_dcc_boosting/sub_model_1
distillation_multiscale_cnnlstm_boosting/sub_model_1
distillchain_lstm_gate_moe8
distillchain_video_dcc
distillchain_cnn_dcc
distillchain_cnndcc_layer2moe4_ensemble
distillchain_lstm_attention8max
distillchain_lstm_moe8
distillchain_lstm_multiscale_layer2_moe8
distillchain_lstm_multiscale_layer4_moe8
lstm_gate_multiscale4_moe4
lstmbiglu_cell1024_layer1_moe8
distillchain_lstmcnn_dcc
distillchain_v2_video_dcc
distillsplit_lstm_gate_moe8
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/ensemble_no2.conf
================================================
cnn_deep_combine_chain
deep_cnn_deep_combine_chain
lstm_attention8_max
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
lstmmemory_cell1024_layer2_moe8
lstmmemory_cell2048_layer2_moe4
lstmparallelmemory1024_moe8
multilstmmemory1024_moe4_deep_chain
video_very_deep_combine_chain
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/ensemble_no20.conf
================================================
cnn_deep_combine_chain
cnn_deep_combine_chain_bagging/ensemble_matrix_model
distillation_cnn_dcc_boosting/sub_model_1
distillation_lstmcnn_dcc_boosting/sub_model_1
distillation_multilstm_dcc_boosting/sub_model_1
framehop_lstm
lstm_attention8_max
lstm_cnn_deep_combine_chain
lstm_positional_attention8max
lstmattention8max_bagging/ensemble_matrix_model
lstmattention8max_boosting/ensemble_attention_matrix_model
lstmbiglu_cell1024_layer1_moe8
lstmglu2_cell1024_layer1_moe8
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
lstmparallelfinaloutput1024_moe8
lstmparallelmemory1024_moe8
multilstmmemory1024_moe4_deep_chain
video_dcc_boosting/ensemble_matrix_model
video_dcc_boosting_discardhopeless/ensemble_matrix_model
video_dcc_boosting_weightclip/ensemble_matrix_model
distillchain_cnn_dcc
distillchain_cnndcc_layer2moe4_ensemble
distillchain_lstm_attention8max
distillchain_lstm_gate_moe8
distillchain_lstm_moe8
distillchain_lstm_multiscale_layer2_moe8
distillchain_lstm_multiscale_layer4_moe8
distillchain_video_dcc
lstm_gate_multiscale4_moe4
distillchain_lstmcnn_dcc
distillchain_v2_video_dcc
distillsplit_lstm_gate_moe8
distillchain_v2_lstmattention8max
distillchain_v2_lstmcnn_dcc
distillchain_v2_lstm_gate_moe8
distillchain_v2_lstm_glu2_moe8
distillchain_v2_multilstm_dcc
distillchain_v2_multiscale_cnnlstm
distillchain_v2_lstmparalleloutput
distillchain_v2_boost_lstmparalleloutput
videochain/distillchain_v2_video_dcc_1
videochain/distillchain_v2_video_dcc_2
videochain/distillchain_v2_video_dcc_3
videochain/distillchain_v2_video_dcc_4
videochain/distillchain_v2_video_dcc_5
hybridchain/distillchain_v2_cnn_dcc
hybridchain/distillchain_v2_lstmparalleloutput
hybridchain/distillchain_v2_video_dcc
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/ensemble_no21.conf
================================================
attentionlstm_moe4
biunilstm1024_moe4
cnn_deep_combine_chain
cnn_deep_combine_chain_bagging/ensemble_matrix_model
cnn_deep_combine_chain_boosting/ensemble_attention_matrix_model
cnnlstmmemory1024_moe8
deep_cnn_deep_combine_chain
distillation_video_dcc_bagging/ensemble_matrix_model
distillation_video_dcc_boosting/ensemble_matrix_model
distillation_video_dcc_noise
framehop_lstm
lstm2_attention8_max
lstm_attention8_max
lstmattention8max_bagging/ensemble_matrix_model
lstmattention8max_boosting/ensemble_attention_matrix_model
lstmattlstm1024_moe8
lstm_cnn_deep_combine_chain
lstmgate_cell1024_layer1_moe8
lstmglu2_cell1024_layer1_moe8
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
lstmmemory_cell1024_layer2_moe8
lstmmemory_cell2048_layer2_moe4
lstm_multiscale4_moe4
lstmparallelfinaloutput1024_moe8
lstmparallelmemory1024_moe8
lstmparalleloutput_bagging/ensemble_mean_model
lstmparalleloutput_boosting_weightclip/ensemble_matrix_model
lstm_positional_attention8max
lstm_random_mean_moe8
lstm_shortlayers_moe8
multilstmmemory1024_moe4_deep_chain
multires_lstm_deep_combine_chain
video_cc_structure_bagging/ensemble_matrix_model
video_dcc_bagging
video_dcc_boosting_discardhopeless/ensemble_matrix_model
video_dcc_boosting/ensemble_matrix_model
video_dcc_boosting_weightclip/ensemble_matrix_model
video_relabel_combine_chain
video_very_deep_combine_chain
distillation_cnn_dcc_boosting/sub_model_1
distillation_lstmcnn_dcc_boosting/sub_model_1
distillation_lstmparalleloutput_boosting/sub_model_1
distillation_multilstm_dcc_boosting/sub_model_1
distillation_multiscale_cnnlstm_boosting/sub_model_1
distillchain_lstm_gate_moe8
distillchain_video_dcc
distillchain_cnn_dcc
distillchain_cnndcc_layer2moe4_ensemble
distillchain_lstm_attention8max
distillchain_lstm_moe8
distillchain_lstm_multiscale_layer2_moe8
distillchain_lstm_multiscale_layer4_moe8
lstm_gate_multiscale4_moe4
lstmbiglu_cell1024_layer1_moe8
distillchain_lstmcnn_dcc
distillchain_v2_video_dcc
distillsplit_lstm_gate_moe8
distillchain_v2_lstmattention8max
distillchain_v2_lstmcnn_dcc
distillchain_v2_lstm_gate_moe8
distillchain_v2_lstm_glu2_moe8
distillchain_v2_multilstm_dcc
distillchain_v2_multiscale_cnnlstm
videochain/distillchain_v2_video_dcc_1
videochain/distillchain_v2_video_dcc_2
videochain/distillchain_v2_video_dcc_3
videochain/distillchain_v2_video_dcc_4
videochain/distillchain_v2_video_dcc_5
hybridchain/distillchain_v2_cnn_dcc
hybridchain/distillchain_v2_lstmparalleloutput
hybridchain/distillchain_v2_video_dcc
distillchain_v2_lstmparalleloutput
distillchain_v2_boost_lstmparalleloutput
distillchain_video_norm_moe8
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/ensemble_no3.conf
================================================
attentionlstm_moe4
biunilstm1024_moe4
cnn_deep_combine_chain
cnnlstmmemory1024_moe8
deep_cnn_deep_combine_chain
framehop_lstm
lstm_attention8_max
lstmattlstm1024_moe8
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
lstmmemory_cell1024_layer2_moe8
lstmmemory_cell2048_layer2_moe4
lstmparallelfinaloutput1024_moe8
lstmparallelmemory1024_moe8
multilstmmemory1024_moe4_deep_chain
video_very_deep_combine_chain
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/ensemble_no4.conf
================================================
attentionlstm_moe4
biunilstm1024_moe4
cnn_deep_combine_chain
cnnlstmmemory1024_moe8
deep_cnn_deep_combine_chain
framehop_lstm
lstm_attention8_max
lstmattlstm1024_moe8
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
lstmmemory_cell1024_layer2_moe8
lstmmemory_cell2048_layer2_moe4
lstmparallelfinaloutput1024_moe8
lstmparallelmemory1024_moe8
multilstmmemory1024_moe4_deep_chain
video_very_deep_combine_chain
lstm_shortlayers_moe8
video_dcc_bagging
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/ensemble_no5.conf
================================================
cnn_deep_combine_chain
cnnlstmmemory1024_moe8
deep_cnn_deep_combine_chain
framehop_lstm
lstm_attention8_max
lstm_cnn_deep_combine_chain
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
lstmparallelfinaloutput1024_moe8
lstmparallelmemory1024_moe8
multilstmmemory1024_moe4_deep_chain
multires_lstm_deep_combine_chain
video_dcc_bagging
video_very_deep_combine_chain
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/ensemble_no6.conf
================================================
attentionlstm_moe4
biunilstm1024_moe4
cnn_deep_combine_chain
cnnlstmmemory1024_moe8
deep_cnn_deep_combine_chain
framehop_lstm
lstm_attention8_max
lstmattlstm1024_moe8
lstm_cnn_deep_combine_chain
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
lstmmemory_cell1024_layer2_moe8
lstmmemory_cell2048_layer2_moe4
lstmparallelfinaloutput1024_moe8
lstmparallelmemory1024_moe8
lstm_shortlayers_moe8
multilstmmemory1024_moe4_deep_chain
multires_lstm_deep_combine_chain
video_dcc_bagging
video_very_deep_combine_chain
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/ensemble_no7.conf
================================================
attentionlstm_moe4
biunilstm1024_moe4
cnn_deep_combine_chain
cnnlstmmemory1024_moe8
deep_cnn_deep_combine_chain
framehop_lstm
lstm2_attention8_max
lstm_attention8_max
lstm_random_mean_moe8
lstmattlstm1024_moe8
lstm_cnn_deep_combine_chain
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
lstmmemory_cell1024_layer2_moe8
lstmmemory_cell2048_layer2_moe4
lstmparallelfinaloutput1024_moe8
lstmparallelmemory1024_moe8
lstm_shortlayers_moe8
multilstmmemory1024_moe4_deep_chain
multires_lstm_deep_combine_chain
video_very_deep_combine_chain
video_dcc_bagging
video_cc_structure_bagging/ensemble_matrix_model
video_dcc_boosting/ensemble_matrix_model
video_dcc_boosting_discardhopeless/ensemble_matrix_model
video_dcc_boosting_weightclip/ensemble_matrix_model
cnn_deep_combine_chain_bagging/ensemble_matrix_model
lstmparalleloutput_bagging/ensemble_mean_model
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/ensemble_no8.conf
================================================
cnn_deep_combine_chain
cnn_deep_combine_chain_bagging/ensemble_matrix_model
cnnlstmmemory1024_moe8
framehop_lstm
lstm_attention8_max
lstm_cnn_deep_combine_chain
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
lstmparallelfinaloutput1024_moe8
lstmparallelmemory1024_moe8
multilstmmemory1024_moe4_deep_chain
multires_lstm_deep_combine_chain
video_cc_structure_bagging/ensemble_matrix_model
video_dcc_boosting/ensemble_matrix_model
video_dcc_boosting_discardhopeless/ensemble_matrix_model
video_dcc_boosting_weightclip/ensemble_matrix_model
lstmparalleloutput_bagging/ensemble_mean_model
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/ensemble_no9.conf
================================================
attentionlstm_moe4
biunilstm1024_moe4
cnn_deep_combine_chain
cnnlstmmemory1024_moe8
deep_cnn_deep_combine_chain
framehop_lstm
lstm2_attention8_max
lstm_attention8_max
lstm_random_mean_moe8
lstmattlstm1024_moe8
lstm_cnn_deep_combine_chain
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
lstmmemory_cell1024_layer2_moe8
lstmmemory_cell2048_layer2_moe4
lstmparallelfinaloutput1024_moe8
lstmparallelmemory1024_moe8
lstm_shortlayers_moe8
multilstmmemory1024_moe4_deep_chain
multires_lstm_deep_combine_chain
video_very_deep_combine_chain
video_relabel_combine_chain
video_dcc_bagging
video_cc_structure_bagging/ensemble_matrix_model
video_dcc_boosting/ensemble_matrix_model
video_dcc_boosting_discardhopeless/ensemble_matrix_model
video_dcc_boosting_weightclip/ensemble_matrix_model
cnn_deep_combine_chain_bagging/ensemble_matrix_model
lstmparalleloutput_bagging/ensemble_mean_model
lstmattention8max_bagging/ensemble_matrix_model
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/eval-attention_linear_model.sh
================================================
model=$1
conf=$2
start=$3
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
validate_path=/Youtube-8M/model_predictions/ensemble_validate
validate_data_patterns=""
for d in $(cat $conf); do
validate_data_patterns="${validate_path}/${d}/*.tfrecord${validate_data_patterns:+,$validate_data_patterns}"
done
echo "$validate_data_patterns"
input_data_pattern="${validate_path}/model_input/*.tfrecord"
EVERY=300
DIR="$(pwd)"
MODEL_DIR="${DIR}/../model/${model}" \
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES="$GPU_ID" python eval.py \
--model_checkpoint_path="../model/${model}/model.ckpt-${checkpoint}" \
--train_dir="../model/${model}" \
--model="AttentionLinearModel" \
--moe_num_mixtures=16 \
--eval_data_patterns="$validate_data_patterns" \
--input_data_pattern="$input_data_pattern"
fi
done
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/eval-attention_linmatrix_model.sh
================================================
model=$1
conf=$2
start=$3
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
validate_path=/Youtube-8M/model_predictions/ensemble_validate
validate_data_patterns=""
for d in $(cat $conf); do
validate_data_patterns="${validate_path}/${d}/*.tfrecord${validate_data_patterns:+,$validate_data_patterns}"
done
echo "$validate_data_patterns"
input_data_pattern="${validate_path}/model_input/*.tfrecord"
EVERY=100
DIR="$(pwd)"
MODEL_DIR="${DIR}/../model/${model}" \
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES="$GPU_ID" python eval.py \
--model_checkpoint_path="../model/${model}/model.ckpt-${checkpoint}" \
--train_dir="../model/${model}" \
--model="AttentionLinmatrixModel" \
--moe_num_mixtures=16 \
--attention_matrix_rank=4 \
--batch_size=1024 \
--eval_data_patterns="$validate_data_patterns" \
--input_data_pattern="$input_data_pattern"
fi
done
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/eval-attention_matrix_model.sh
================================================
model=$1
conf=$2
start=$3
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
validate_path=/Youtube-8M/model_predictions/ensemble_validate
validate_data_patterns=""
for d in $(cat $conf); do
validate_data_patterns="${validate_path}/${d}/*.tfrecord${validate_data_patterns:+,$validate_data_patterns}"
done
echo "$validate_data_patterns"
input_data_pattern="${validate_path}/model_input/*.tfrecord"
EVERY=200
DIR="$(pwd)"
MODEL_DIR="${DIR}/../model/${model}" \
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES="$GPU_ID" python eval.py \
--model_checkpoint_path="../model/${model}/model.ckpt-${checkpoint}" \
--train_dir="../model/${model}" \
--model="AttentionMatrixModel" \
--moe_num_mixtures=4 \
--attention_matrix_rank=8 \
--eval_data_patterns="$validate_data_patterns" \
--input_data_pattern="$input_data_pattern"
fi
done
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/eval-attention_moe_matrix_model.sh
================================================
model=$1
conf=$2
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
validate_path=/Youtube-8M/model_predictions/ensemble_validate
validate_data_patterns=""
for d in $(cat $conf); do
validate_data_patterns="${validate_path}/${d}/*.tfrecord${validate_data_patterns:+,$validate_data_patterns}"
done
echo "$validate_data_patterns"
input_data_pattern="${validate_path}/model_input/*.tfrecord"
EVERY=200
start=0
DIR="$(pwd)"
MODEL_DIR="${DIR}/../model/${model}" \
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES="$GPU_ID" python eval.py \
--model_checkpoint_path="../model/${model}/model.ckpt-${checkpoint}" \
--train_dir="../model/${model}" \
--model="AttentionMatrixModel" \
--moe_num_mixtures=4 \
--attention_relu_cells=256 \
--attention_matrix_rank=4 \
--eval_data_patterns="$validate_data_patterns" \
--input_data_pattern="$input_data_pattern"
fi
done
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/eval-attention_moe_model.sh
================================================
model=$1
conf=$2
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
validate_path=/Youtube-8M/model_predictions/ensemble_validate
validate_data_patterns=""
for d in $(cat $conf); do
validate_data_patterns="${validate_path}/${d}/*.tfrecord${validate_data_patterns:+,$validate_data_patterns}"
done
echo "$validate_data_patterns"
input_data_pattern="${validate_path}/model_input/*.tfrecord"
EVERY=200
start=0
DIR="$(pwd)"
MODEL_DIR="${DIR}/../model/${model}" \
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES="$GPU_ID" python eval.py \
--model_checkpoint_path="../model/${model}/model.ckpt-${checkpoint}" \
--train_dir="../model/${model}" \
--model=AttentionMoeModel \
--moe_num_mixtures=8 \
--attention_relu_cells=128 \
--eval_data_patterns="$validate_data_patterns" \
--input_data_pattern="$input_data_pattern"
fi
done
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/eval-attention_rectified_linear_model.sh
================================================
model=$1
conf=$2
start=$3
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
validate_path=/Youtube-8M/model_predictions/ensemble_validate
validate_data_patterns=""
for d in $(cat $conf); do
validate_data_patterns="${validate_path}/${d}/*.tfrecord${validate_data_patterns:+,$validate_data_patterns}"
done
echo "$validate_data_patterns"
input_data_pattern="${validate_path}/model_input/*.tfrecord"
EVERY=300
DIR="$(pwd)"
MODEL_DIR="${DIR}/../model/${model}" \
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES="$GPU_ID" python eval.py \
--model_checkpoint_path="../model/${model}/model.ckpt-${checkpoint}" \
--train_dir="../model/${model}" \
--model="AttentionRectifiedLinearModel" \
--moe_num_mixtures=16 \
--eval_data_patterns="$validate_data_patterns" \
--input_data_pattern="$input_data_pattern"
fi
done
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/eval-deep_combine_chain_model.sh
================================================
model=$1
conf=$2
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
validate_path=/Youtube-8M/model_predictions/ensemble_validate
validate_data_patterns=""
labels=""
for d in $(cat $conf); do
validate_data_patterns="${validate_path}/${d}/*.tfrecord${validate_data_patterns:+,$validate_data_patterns}"
labels="label,${labels:+,$labels}"
done
echo "$validate_data_patterns"
input_data_pattern="${validate_path}/model_input/*.tfrecord"
EVERY=100
start=-1
DIR="$(pwd)"
MODEL_DIR="${DIR}/../model/${model}" \
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES="$GPU_ID" python eval.py \
--model_checkpoint_path="../model/${model}/model.ckpt-${checkpoint}" \
--train_dir="../model/${model}" \
--model="DeepCombineChainModel" \
--deep_chain_relu_cells=128 \
--moe_num_mixtures=2 \
--eval_data_patterns="$validate_data_patterns" \
--batch_size=1024 \
--input_data_pattern="$input_data_pattern"
fi
done
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/eval-input_moe_model.sh
================================================
model=$1
conf=$2
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
validate_path=/Youtube-8M/model_predictions/ensemble_validate
validate_data_patterns=""
for d in $(cat $conf); do
validate_data_patterns="${validate_path}/${d}/*.tfrecord${validate_data_patterns:+,$validate_data_patterns}"
done
echo "$validate_data_patterns"
input_data_pattern="${validate_path}/model_input/*.tfrecord"
EVERY=100
start=-1
DIR="$(pwd)"
MODEL_DIR="${DIR}/../model/${model}" \
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES="$GPU_ID" python eval.py \
--model_checkpoint_path="../model/${model}/model.ckpt-${checkpoint}" \
--train_dir="../model/${model}" \
--input_data_pattern=$input_data_pattern \
--model="InputMoeModel" \
--eval_data_patterns="$validate_data_patterns"
fi
done
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/eval-linear_model.sh
================================================
model=$1
conf=$2
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
validate_path=/Youtube-8M/model_predictions/ensemble_validate
validate_data_patterns=""
for d in $(cat $conf); do
validate_data_patterns="${validate_path}/${d}/*.tfrecord${validate_data_patterns:+,$validate_data_patterns}"
done
echo "$validate_data_patterns"
EVERY=100
start=-1
DIR="$(pwd)"
MODEL_DIR="${DIR}/../model/${model}" \
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES="$GPU_ID" python eval.py \
--model_checkpoint_path="../model/${model}/model.ckpt-${checkpoint}" \
--train_dir="../model/${model}" \
--model="LinearRegressionModel" \
--eval_data_patterns="$validate_data_patterns"
fi
done
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/eval-matrix_model.sh
================================================
model=$1
conf=$2
postfix=$3
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
validate_path=/Youtube-8M/model_predictions${postfix}/ensemble_validate
validate_data_patterns=""
for d in $(cat $conf); do
validate_data_patterns="${validate_path}/${d}/*.tfrecord${validate_data_patterns:+,$validate_data_patterns}"
done
echo "$validate_data_patterns"
EVERY=200
start=-1
DIR="$(pwd)"
MODEL_DIR="${DIR}/../model/${model}" \
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES="$GPU_ID" python eval.py \
--model_checkpoint_path="../model/${model}/model.ckpt-${checkpoint}" \
--train_dir="../model/${model}" \
--model="MatrixRegressionModel" \
--eval_data_patterns="$validate_data_patterns"
fi
done
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/eval-mean_model.sh
================================================
model=$1
conf=$2
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
validate_path=/Youtube-8M/model_predictions/ensemble_validate
validate_data_patterns=""
for d in $(cat $conf); do
validate_data_patterns="${validate_path}/${d}/*.tfrecord${validate_data_patterns:+,$validate_data_patterns}"
done
echo "$validate_data_patterns"
CUDA_VISIBLE_DEVICES="$GPU_ID" python eval.py \
--model_checkpoint_path="../model/${model}/model.ckpt-0" \
--train_dir="../model/${model}" \
--model="MeanModel" \
--eval_data_patterns="$validate_data_patterns"
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/eval-moe_model.sh
================================================
model=$1
conf=$2
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
validate_path=/Youtube-8M/model_predictions/ensemble_validate
validate_data_patterns=""
for d in $(cat $conf); do
validate_data_patterns="${validate_path}/${d}/*.tfrecord${validate_data_patterns:+,$validate_data_patterns}"
done
echo "$validate_data_patterns"
EVERY=100
start=-1
DIR="$(pwd)"
MODEL_DIR="${DIR}/../model/${model}" \
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES="$GPU_ID" python eval.py \
--model_checkpoint_path="../model/${model}/model.ckpt-${checkpoint}" \
--train_dir="../model/${model}" \
--model="MoeModel" \
--eval_data_patterns="$validate_data_patterns"
fi
done
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/eval-nonunit_matrix_model.sh
================================================
model=$1
conf=$2
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
validate_path=/Youtube-8M/model_predictions/ensemble_validate
validate_data_patterns=""
for d in $(cat $conf); do
validate_data_patterns="${validate_path}/${d}/*.tfrecord${validate_data_patterns:+,$validate_data_patterns}"
done
echo "$validate_data_patterns"
EVERY=100
start=-1
DIR="$(pwd)"
MODEL_DIR="${DIR}/../model/${model}" \
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES="$GPU_ID" python eval.py \
--model_checkpoint_path="../model/${model}/model.ckpt-${checkpoint}" \
--train_dir="../model/${model}" \
--model="NonunitMatrixRegressionModel" \
--eval_data_patterns="$validate_data_patterns"
fi
done
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/explore-mean_model.log
================================================
single model
------------------------------------------------
cnn_deep_combine_chain
GAP = 0.816665221681
------------------------------------------------
lstm_attention8_max
GAP = 0.815108872388
------------------------------------------------
lstmparallelmemory1024_moe8
GAP = 0.814833331691
------------------------------------------------
deep_cnn_deep_combine_chain
GAP = 0.814806325097
------------------------------------------------
lstmmem1024_layer2_moe4_deep_combine_chain_add_length
GAP = 0.814129286481
------------------------------------------------
lstmmemory_cell2048_layer2_moe4
GAP = 0.813540507351
------------------------------------------------
lstmmemory_cell1024_layer2_moe8
GAP = 0.812080163135
------------------------------------------------
video_very_deep_combine_chain
GAP = 0.810174420122
------------------------------------------------
2-model ensemble
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain
GAP = 0.82557031245
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max
GAP = 0.829659120727
------------------------------------------------
cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length
GAP = 0.828572770795
------------------------------------------------
cnn_deep_combine_chain,lstmmemory_cell1024_layer2_moe8
GAP = 0.827447547713
------------------------------------------------
cnn_deep_combine_chain,lstmmemory_cell2048_layer2_moe4
GAP = 0.828015668957
------------------------------------------------
cnn_deep_combine_chain,lstmparallelmemory1024_moe8
GAP = 0.829346968501
------------------------------------------------
cnn_deep_combine_chain,video_very_deep_combine_chain
GAP = 0.825294576305
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max
GAP = 0.827991520743
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length
GAP = 0.827050978978
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmemory_cell1024_layer2_moe8
GAP = 0.825639821236
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmemory_cell2048_layer2_moe4
GAP = 0.826148618017
------------------------------------------------
deep_cnn_deep_combine_chain,lstmparallelmemory1024_moe8
GAP = 0.827756382817
------------------------------------------------
deep_cnn_deep_combine_chain,video_very_deep_combine_chain
GAP = 0.823699502378
------------------------------------------------
lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length
GAP = 0.827966037686
------------------------------------------------
lstm_attention8_max,lstmmemory_cell1024_layer2_moe8
GAP = 0.82594497815
------------------------------------------------
lstm_attention8_max,lstmmemory_cell2048_layer2_moe4
GAP = 0.826350996945
------------------------------------------------
lstm_attention8_max,lstmparallelmemory1024_moe8
GAP = 0.828111399229
------------------------------------------------
lstm_attention8_max,video_very_deep_combine_chain
GAP = 0.827831471962
------------------------------------------------
lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8
GAP = 0.824264654816
------------------------------------------------
lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4
GAP = 0.824553122942
------------------------------------------------
lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmparallelmemory1024_moe8
GAP = 0.826524738997
------------------------------------------------
lstmmem1024_layer2_moe4_deep_combine_chain_add_length,video_very_deep_combine_chain
GAP = 0.826541745861
------------------------------------------------
lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4
GAP = 0.822700768554
------------------------------------------------
lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8
GAP = 0.824428614492
------------------------------------------------
lstmmemory_cell1024_layer2_moe8,video_very_deep_combine_chain
GAP = 0.825377747168
------------------------------------------------
lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.825320116688
------------------------------------------------
lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.825870841013
------------------------------------------------
lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.827311197228
------------------------------------------------
3-model ensemble
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max
GAP = 0.832191332449
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length
GAP = 0.831426452795
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmemory_cell1024_layer2_moe8
GAP = 0.830459884161
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmemory_cell2048_layer2_moe4
GAP = 0.830804693389
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmparallelmemory1024_moe8
GAP = 0.832026266634
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,video_very_deep_combine_chain
GAP = 0.828845214499
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length
GAP = 0.833621054469
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell1024_layer2_moe8
GAP = 0.832483129822
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell2048_layer2_moe4
GAP = 0.832721553048
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmparallelmemory1024_moe8
GAP = 0.834000141252
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,video_very_deep_combine_chain
GAP = 0.832594974463
------------------------------------------------
cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8
GAP = 0.83140605405
------------------------------------------------
cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4
GAP = 0.831612457968
------------------------------------------------
cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmparallelmemory1024_moe8
GAP = 0.832955557705
------------------------------------------------
cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,video_very_deep_combine_chain
GAP = 0.831666234693
------------------------------------------------
cnn_deep_combine_chain,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4
GAP = 0.830441764733
------------------------------------------------
cnn_deep_combine_chain,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8
GAP = 0.831766254279
------------------------------------------------
cnn_deep_combine_chain,lstmmemory_cell1024_layer2_moe8,video_very_deep_combine_chain
GAP = 0.830901832056
------------------------------------------------
cnn_deep_combine_chain,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.832169199739
------------------------------------------------
cnn_deep_combine_chain,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.831109764339
------------------------------------------------
cnn_deep_combine_chain,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.83223587122
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length
GAP = 0.832349778772
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell1024_layer2_moe8
GAP = 0.831052849226
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell2048_layer2_moe4
GAP = 0.831305454699
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmparallelmemory1024_moe8
GAP = 0.832756750266
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,video_very_deep_combine_chain
GAP = 0.831369689002
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8
GAP = 0.829913304808
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4
GAP = 0.830247896269
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmparallelmemory1024_moe8
GAP = 0.831736842166
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,video_very_deep_combine_chain
GAP = 0.830453715972
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4
GAP = 0.828878078355
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8
GAP = 0.830298620071
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmemory_cell1024_layer2_moe8,video_very_deep_combine_chain
GAP = 0.829553175272
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.830743752891
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.829768283623
------------------------------------------------
deep_cnn_deep_combine_chain,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.831023711068
------------------------------------------------
lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8
GAP = 0.830592277718
------------------------------------------------
lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4
GAP = 0.830746937407
------------------------------------------------
lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmparallelmemory1024_moe8
GAP = 0.832236858925
------------------------------------------------
lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,video_very_deep_combine_chain
GAP = 0.832650348923
------------------------------------------------
lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4
GAP = 0.829139497202
------------------------------------------------
lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8
GAP = 0.830648119982
------------------------------------------------
lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,video_very_deep_combine_chain
GAP = 0.831435923278
------------------------------------------------
lstm_attention8_max,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.831018565226
------------------------------------------------
lstm_attention8_max,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.831602818518
------------------------------------------------
lstm_attention8_max,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.833058032219
------------------------------------------------
lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4
GAP = 0.827740923928
------------------------------------------------
lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8
GAP = 0.829327455006
------------------------------------------------
lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,video_very_deep_combine_chain
GAP = 0.830225595389
------------------------------------------------
lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.829626608004
------------------------------------------------
lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.830460642345
------------------------------------------------
lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.831778438641
------------------------------------------------
lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.828104386062
------------------------------------------------
lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.829167967486
------------------------------------------------
lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.830592592778
------------------------------------------------
lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.830995957639
------------------------------------------------
4-model ensemble
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length
GAP = 0.834859637855
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell1024_layer2_moe8
GAP = 0.833895260202
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell2048_layer2_moe4
GAP = 0.834045908801
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmparallelmemory1024_moe8
GAP = 0.835235503218
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,video_very_deep_combine_chain
GAP = 0.833723731659
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8
GAP = 0.833104488978
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4
GAP = 0.833324163897
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmparallelmemory1024_moe8
GAP = 0.834484898844
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,video_very_deep_combine_chain
GAP = 0.832930582133
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4
GAP = 0.832377724469
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8
GAP = 0.833414957585
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmemory_cell1024_layer2_moe8,video_very_deep_combine_chain
GAP = 0.832301392579
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.833769221192
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.832408546437
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.833437630091
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8
GAP = 0.834464961259
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4
GAP = 0.834659214213
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmparallelmemory1024_moe8
GAP = 0.835789192715
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,video_very_deep_combine_chain
GAP = 0.835273660124
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4
GAP = 0.833592380025
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8
GAP = 0.834699040434
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,video_very_deep_combine_chain
GAP = 0.834399178828
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.834979500288
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.834599103937
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.835657070379
------------------------------------------------
cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4
GAP = 0.832603654742
------------------------------------------------
cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8
GAP = 0.8337961468
------------------------------------------------
cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,video_very_deep_combine_chain
GAP = 0.833522575937
------------------------------------------------
cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.833995288321
------------------------------------------------
cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.833691549239
------------------------------------------------
cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.834677051398
------------------------------------------------
cnn_deep_combine_chain,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.832988258644
------------------------------------------------
cnn_deep_combine_chain,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.83279736014
------------------------------------------------
cnn_deep_combine_chain,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.833836412621
------------------------------------------------
cnn_deep_combine_chain,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.834129839666
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8
GAP = 0.833393442149
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4
GAP = 0.833546049331
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmparallelmemory1024_moe8
GAP = 0.834839753828
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,video_very_deep_combine_chain
GAP = 0.83420746933
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4
GAP = 0.83240292504
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8
GAP = 0.833633371649
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,video_very_deep_combine_chain
GAP = 0.833366664258
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.833897307363
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.833519613265
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.834663565256
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4
GAP = 0.831446232504
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8
GAP = 0.832594788307
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,video_very_deep_combine_chain
GAP = 0.832457962265
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.832881953446
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.832662090456
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.833702886165
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.831723349274
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.831664565904
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.832780128708
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.83309748339
------------------------------------------------
lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4
GAP = 0.831784972382
------------------------------------------------
lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8
GAP = 0.83302089599
------------------------------------------------
lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,video_very_deep_combine_chain
GAP = 0.833818492121
------------------------------------------------
lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.833296927263
------------------------------------------------
lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.833948837411
------------------------------------------------
lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.835128538546
------------------------------------------------
lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.831896833011
------------------------------------------------
lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.832843246048
------------------------------------------------
lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.833988784393
------------------------------------------------
lstm_attention8_max,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.834228084116
------------------------------------------------
lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.830787155159
------------------------------------------------
lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.831791382924
------------------------------------------------
lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.832990647204
------------------------------------------------
lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.833219017873
------------------------------------------------
lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.832099590832
------------------------------------------------
5-model ensemble
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8
GAP = 0.835438033957
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4
GAP = 0.835605498079
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmparallelmemory1024_moe8
GAP = 0.83653394275
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,video_very_deep_combine_chain
GAP = 0.835745783944
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4
GAP = 0.834686137206
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8
GAP = 0.835670112826
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,video_very_deep_combine_chain
GAP = 0.835047895241
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.835889967275
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.835210150381
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.836093418248
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4
GAP = 0.83401135364
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8
GAP = 0.834938997204
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,video_very_deep_combine_chain
GAP = 0.834311398082
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.835178958848
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.834508517888
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.835322108623
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.834318844694
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.833795003243
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.834649121681
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.834845499789
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4
GAP = 0.83493975204
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8
GAP = 0.835876294898
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,video_very_deep_combine_chain
GAP = 0.835856824821
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.836099640517
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.836003869651
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.836959694784
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.835127020411
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.835186988207
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.836084071621
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.836365387451
------------------------------------------------
cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.834314811777
------------------------------------------------
cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.834380440941
------------------------------------------------
cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.835262275125
------------------------------------------------
cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.835462501109
------------------------------------------------
cnn_deep_combine_chain,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.834682844519
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4
GAP = 0.834000181534
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8
GAP = 0.834996923723
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,video_very_deep_combine_chain
GAP = 0.835026558331
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.835168142367
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.835154802734
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.836113458024
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.834178068594
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.834322025109
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.835242509217
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.835421384341
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.833302862578
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.833484620513
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.834398472225
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.834618836206
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.833736799505
------------------------------------------------
lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.833542783923
------------------------------------------------
lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.834372993093
------------------------------------------------
lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.835390081231
------------------------------------------------
lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.835597107314
------------------------------------------------
lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.83455284016
------------------------------------------------
lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.833675261416
------------------------------------------------
6-model ensemble
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4
GAP = 0.835764820291
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8
GAP = 0.836573100548
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,video_very_deep_combine_chain
GAP = 0.836286868542
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.836789577149
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.836410907788
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.837230111667
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.835967761151
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.835742351815
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.836527025619
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.836708452803
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.835354805374
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.835042792794
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.835826695583
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.836076030996
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.835345677749
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.835999247145
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.836183473417
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.837008192923
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.837117755989
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.836389579777
------------------------------------------------
cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.835662725883
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.835175538634
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.835437516935
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.836246224116
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.83637430754
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.835617024019
------------------------------------------------
deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.834914528567
------------------------------------------------
lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.835543683299
------------------------------------------------
7-model ensemble
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8
GAP = 0.836664222772
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,video_very_deep_combine_chain
GAP = 0.83658908908
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.837284897399
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.837397359344
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.836838205684
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.836228065763
------------------------------------------------
cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.837045884874
------------------------------------------------
deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.836382105823
------------------------------------------------
8-model ensemble
------------------------------------------------
cnn_deep_combine_chain,deep_cnn_deep_combine_chain,lstm_attention8_max,lstmmem1024_layer2_moe4_deep_combine_chain_add_length,lstmmemory_cell1024_layer2_moe8,lstmmemory_cell2048_layer2_moe4,lstmparallelmemory1024_moe8,video_very_deep_combine_chain
GAP = 0.837375440061
------------------------------------------------
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/explore-mean_model.sh
================================================
train_path=/Youtube-8M/model_predictions/ensemble_train
for d in $(ls $train_path | sort); do
train_data_patterns="${train_path}/${d}/*.tfrecord"
echo "$d"
CUDA_VISIBLE_DEVICES=0 python eval.py \
--model_checkpoint_path="../model/mean_model/model.ckpt-0" \
--train_dir="../model/mean_model" \
--model="MeanModel" \
--echo_gap=True \
--eval_data_patterns=$train_data_patterns | tail -n 1
echo "------------------------------------------------"
done
for d1 in $(ls $train_path | sort); do
for d2 in $(ls $train_path | sort); do
if [[ "$d1" < "$d2" ]]; then
train_data_patterns="${train_path}/${d1}/*.tfrecord,${train_path}/${d2}/*.tfrecord"
echo "$d1,$d2"
CUDA_VISIBLE_DEVICES=0 python eval.py \
--model_checkpoint_path="../model/mean_model/model.ckpt-0" \
--train_dir="../model/mean_model" \
--model="MeanModel" \
--echo_gap=True \
--eval_data_patterns=$train_data_patterns | tail -n 1
echo "------------------------------------------------"
fi
done
done
for d1 in $(ls $train_path | sort); do
for d2 in $(ls $train_path | sort); do
if [[ "$d1" < "$d2" ]]; then
for d3 in $(ls $train_path | sort); do
if [[ "$d2" < "$d3" ]]; then
train_data_patterns="${train_path}/${d1}/*.tfrecord,${train_path}/${d2}/*.tfrecord,${train_path}/${d3}/*.tfrecord"
echo "$d1,$d2,$d3"
CUDA_VISIBLE_DEVICES=0 python eval.py \
--model_checkpoint_path="../model/mean_model/model.ckpt-0" \
--train_dir="../model/mean_model" \
--model="MeanModel" \
--echo_gap=True \
--eval_data_patterns=$train_data_patterns | tail -n 1
echo "------------------------------------------------"
fi
done
fi
done
done
for d1 in $(ls $train_path | sort); do
for d2 in $(ls $train_path | sort); do
if [[ "$d1" < "$d2" ]]; then
for d3 in $(ls $train_path | sort); do
if [[ "$d2" < "$d3" ]]; then
for d4 in $(ls $train_path | sort); do
if [[ "$d3" < "$d4" ]]; then
train_data_patterns="${train_path}/${d1}/*.tfrecord,${train_path}/${d2}/*.tfrecord,${train_path}/${d3}/*.tfrecord,${train_path}/${d4}/*.tfrecord"
echo "$d1,$d2,$d3,$d4"
CUDA_VISIBLE_DEVICES=0 python eval.py \
--model_checkpoint_path="../model/mean_model/model.ckpt-0" \
--train_dir="../model/mean_model" \
--model="MeanModel" \
--echo_gap=True \
--eval_data_patterns=$train_data_patterns | tail -n 1
echo "------------------------------------------------"
fi
done
fi
done
fi
done
done
for d1 in $(ls $train_path | sort); do
for d2 in $(ls $train_path | sort); do
if [[ "$d1" < "$d2" ]]; then
for d3 in $(ls $train_path | sort); do
if [[ "$d2" < "$d3" ]]; then
for d4 in $(ls $train_path | sort); do
if [[ "$d3" < "$d4" ]]; then
for d5 in $(ls $train_path | sort); do
if [[ "$d4" < "$d5" ]]; then
train_data_patterns="${train_path}/${d1}/*.tfrecord,${train_path}/${d2}/*.tfrecord,${train_path}/${d3}/*.tfrecord,${train_path}/${d4}/*.tfrecord,${train_path}/${d5}/*.tfrecord"
echo "$d1,$d2,$d3,$d4,$d5"
CUDA_VISIBLE_DEVICES=0 python eval.py \
--model_checkpoint_path="../model/mean_model/model.ckpt-0" \
--train_dir="../model/mean_model" \
--model="MeanModel" \
--echo_gap=True \
--eval_data_patterns=$train_data_patterns | tail -n 1
echo "------------------------------------------------"
fi
done
fi
done
fi
done
fi
done
done
for d1 in $(ls $train_path | sort); do
for d2 in $(ls $train_path | sort); do
if [[ "$d1" < "$d2" ]]; then
for d3 in $(ls $train_path | sort); do
if [[ "$d2" < "$d3" ]]; then
for d4 in $(ls $train_path | sort); do
if [[ "$d3" < "$d4" ]]; then
for d5 in $(ls $train_path | sort); do
if [[ "$d4" < "$d5" ]]; then
for d6 in $(ls $train_path | sort); do
if [[ "$d5" < "$d6" ]]; then
train_data_patterns="${train_path}/${d1}/*.tfrecord,${train_path}/${d2}/*.tfrecord,${train_path}/${d3}/*.tfrecord,${train_path}/${d4}/*.tfrecord,${train_path}/${d5}/*.tfrecord,${train_path}/${d6}/*.tfrecord"
echo "$d1,$d2,$d3,$d4,$d5,$d6"
CUDA_VISIBLE_DEVICES=0 python eval.py \
--model_checkpoint_path="../model/mean_model/model.ckpt-0" \
--train_dir="../model/mean_model" \
--model="MeanModel" \
--echo_gap=True \
--eval_data_patterns=$train_data_patterns | tail -n 1
echo "------------------------------------------------"
fi
done
fi
done
fi
done
fi
done
fi
done
done
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/final_submission.conf
================================================
# This is not a real configuration file that you could use with ensemble_scripts
# This is merely for the sake of understanding
# The final configuration is ensemble_no21.conf, use it instead
# video baseline models
1. video_relabel_combine_chain
video level combine chain model, add the most confident tag predicted by the current model to the labels when loss<10.0
youtube-8m-zhangteng.video_level_models.MoeMix4Model
#moe_mixtures is 4, #combination_layers is 3, #chain_relu_cells is 100
GAP in validate1 is 0.8088 at step 14863
2. video_very_deep_combine_chain
# lstm baseline models
3. lstmmemory_cell1024_layer2_moe8
4. lstmmemory_cell2048_layer2_moe4
5. lstmparallelfinaloutput1024_moe8
6. lstmparallelmemory1024_moe8
7. lstmgate_cell1024_layer1_moe8
lstm and moe model, using scalar input and forget gate in lstm cell
youtube-8m-zhangteng.frame_level_models.LstmGateModel
#moe_mixtures is 8, #lstm_cells is 1024, #lstm_layers is 1
GAP in validate1 is 0.8148 at step 151107
8. lstmglu2_cell1024_layer1_moe8
lstm and moe model, using scalar input and forget gate in lstm cell, add forward immediate input memory (with its own input and forget gate) to lstm cell
youtube-8m-zhangteng.frame_level_models.LstmGlu2Model
#moe_mixtures is 8, #lstm_cells is 1024, #lstm_layers is 1
GAP in validate1 is 0.8152 at step 132334
9. lstmbiglu_cell1024_layer1_moe8
lstm and moe model, using scalar input and forget gate in lstm cell, add bi-directional immediate input memory (with its own input and forget gate) to lstm cell
youtube-8m-zhangteng.frame_level_models.LstmBigluModel
#moe_mixtures is 8, #lstm_cells is 1024, #lstm_layers is 1
GAP in validate1 is 0.8131 at step 130983
10. biunilstm1024_moe4
11. lstm_random_mean_moe8
lstm and moe model, randomly select half of all frames to create new samples to train
youtube-8m-zhangteng.frame_level_models.LstmRandomModel + youtube-8m-zhangteng.video_level_models.MoeModel
#moe_mixtures is 8, #lstm_cells is 1024, #lstm_layers is 2, augmentation times for each sample is 16 when eval and inference
GAP in validate1 is 0.8145 at step 145850
12. lstm_shortlayers_moe8
lstm and moe model, divide sample into several clips with fixed frames, use one lstm to model intra-clip structure, use another lstm to model inter-clip structure
youtube-8m-zhangteng.frame_level_models.LstmLayerModel + youtube-8m-zhangteng.video_level_models.MoeModel
#moe_mixtures is 8, #lstm_cells is 1024, #lstm_layers is 2, #clip_length_frames is 10
GAP in validate1 is 0.8145 at step 107188
13. framehop_lstm
14. cnnlstmmemory1024_moe8
# attention models
15. attentionlstm_moe4
16. lstmattlstm1024_moe8
17. lstm_attention8_max
lstm and moe model, use frame inputs and lstm outputs to calculate weight of each frame, then get the attention vector using a weighted sum of lstm outputs, then make predictions using moe model, repeat several times, take the maximum prediction of these times as the final prediction
youtube-8m-zhangteng.frame_level_models.LstmExtendModel + youtube-8m-zhangteng.video_level_models.MoeExtendModel
#moe_mixtures is 8, #lstm_cells is 1024, #lstm_layers is 2, #attention_repeat_times is 8
GAP in validate1 is 0.8177 at step 181785
18. lstm2_attention8_max
lstm and moe model, use frame inputs and one lstm outputs to calculate weight of each frame, then get the attention vector using a weighted sum of another lstm outputs, then make predictions using moe model, repeat several times, take the maximum prediction of these times as the final prediction
youtube-8m-zhangteng.frame_level_models.InputExtendModel + youtube-8m-zhangteng.video_level_models.MoeExtendModel
#moe_mixtures is 8, #lstm_cells is 1024, #lstm_layers is 2, #attention_repeat_times is 4
GAP in validate1 is 0.8106 at step 197005
19. lstm_positional_attention8max
# multiple sub model joined by deep combine chain
20. cnn_deep_combine_chain
21. deep_cnn_deep_combine_chain
22. lstm_cnn_deep_combine_chain
23. multilstmmemory1024_moe4_deep_chain
24. lstmmem1024_layer2_moe4_deep_combine_chain_add_length
25. multires_lstm_deep_combine_chain
26. lstm_gate_multiscale4_moe4
cnn-lstm and moe model, use a deep cnn to get multiscale representions at different layers, then use a lstm described in model #7 and moe model at each layer to make predictions, take the mean prediction of these layers as the final prediction
youtube-8m-zhangteng.frame_level_models.LstmMultiscale2Model
#moe_mixtures is 4, #cnn_layers is 4, #cnn_cells is [256,256,512], #cnn_fields is [1,2,3] , #lstm_cells is 1024, #lstm_layers is 1
GAP in validate1 is 0.8166 at step 193803
27. lstm_multiscale4_moe4
cnn-lstm and moe model, use a deep cnn to get multiscale representions at different layers, then use a basic lstm and moe model at each layer to make predictions, take the mean prediction of these layers as the final prediction
youtube-8m-zhangteng.frame_level_models.LstmMultiscaleModel
#moe_mixtures is 4, #cnn_layers is 4, #cnn_cells is [256,256,512], #cnn_fields is [1,2,3] , #lstm_cells is 1024, #lstm_layers is 1
GAP in validate1 is 0.8223 at step 231916
# bagging and boosting models
28. video_dcc_bagging
29. video_cc_structure_bagging/ensemble_matrix_model
the video_cc_structure_bagging model is composed of four sub models:
29.1 video_notzero_combine_chain
video level combine chain model, use both support_predictions and (1-support_predictions) as label dependence at each combination layer
youtube-8m-zhangteng.video_level_models.MoeMix4Model
#moe_mixtures is 4, #combination_layers is 3, #chain_relu_cells is 100
GAP in validate1 is 0.8099 at step 14668
29.2 video_weight_combine_chain
video level combine chain model, multiply loss of positive labels by 10, while multiply loss of negative labels by 1
youtube-8m-zhangteng.video_level_models.MoeMix4Model
#moe_mixtures is 4, #combination_layers is 3, #chain_relu_cells is 100
GAP in validate1 is 0.8079 at step 17415
29.3 video_knowledge_combine_chain
video level combine chain model, use both support_predictions and top-level verticals of support_predictions as label dependence at each combination layer
youtube-8m-zhangteng.video_level_models.MoeKnowledgeModel
#moe_mixtures is 4, #combination_layers is 3, #chain_relu_cells is 100
GAP in validate1 is 0.8108 at step 9606
29.4 video_softmax_combine_chain
video level combine chain model, use sigmiod model to predict the first 3000 labels, and use softmax model to predict the rest 1716 labels
youtube-8m-zhangteng.video_level_models.MoeSoftmaxModel
#moe_mixtures is 4, #combination_layers is 3, #chain_relu_cells is 100
GAP in validate1 is 0.8102 at step 9501
GAP of ensemble_matrix_model in validate2 is
30. video_dcc_boosting_discardhopeless/ensemble_matrix_model
31. video_dcc_boosting/ensemble_matrix_model
32. video_dcc_boosting_weightclip/ensemble_matrix_model
33. cnn_deep_combine_chain_bagging/ensemble_matrix_model
34. cnn_deep_combine_chain_boosting/ensemble_attention_matrix_model
35. lstmattention8max_bagging/ensemble_matrix_model
36. lstmattention8max_boosting/ensemble_attention_matrix_model
37. lstmparalleloutput_bagging/ensemble_mean_model
38. lstmparalleloutput_boosting_weightclip/ensemble_matrix_model
# distillation models
39. distillation_video_dcc_noise
40. distillation_cnn_dcc_boosting/sub_model_1
41. distillation_lstmcnn_dcc_boosting/sub_model_1
42. distillation_lstmparalleloutput_boosting/sub_model_1
43. distillation_multilstm_dcc_boosting/sub_model_1
44. distillation_multiscale_cnnlstm_boosting/sub_model_1
45. distillation_video_dcc_bagging/ensemble_matrix_model
46. distillation_video_dcc_boosting/ensemble_matrix_model
# cascade models
47. distillchain_video_dcc
48. distillchain_video_norm_moe8
video level moe model, use ditill labels as label dependence, normalize the label dependence with the summation of ditill labels
youtube-8m-zhangteng.video_level_models.MoeDistillChainNorm2Model
#moe_mixtures is 8, #chain_relu_cells is 256
GAP in validate1 is 0.8260 at step 13412
49. distillchain_lstm_gate_moe8
lstm and moe model, use lstm cell described in model #7 in frame level, use ditill labels as label dependence in video level
youtube-8m-zhangteng.frame_level_models.LstmGateModel + youtube-8m-zhangteng.video_level_models.MoeDistillChainModel
#moe_mixtures is 8, #chain_relu_cells is 256, #lstm_cells is 1024, #lstm_layers is 1
GAP in validate1 is 0.8293 at step 63344
50. distillchain_cnn_dcc
51. distillchain_cnndcc_layer2moe4
cnn and deep combine chain model, use ditill labels as label dependence only in the first combination layer
youtube-8m-zhangteng.frame_level_models.CnnDCCDistillChainModel
#moe_mixtures is 4, #chain_relu_cells is 256, #combination_layers=2
GAP in validate1 is 0.8241 at step 188273
52. distillchain_lstm_attention8max
lstm and moe model as described in model #17, use ditill labels as label dependence on each attention output vector
youtube-8m-zhangteng.frame_level_models.LstmExtendModel + youtube-8m-zhangteng.video_level_models.MoeExtendDistillChainModel
#moe_mixtures is 8, #chain_relu_cells is 256, #attention_repeat_times is 8, #lstm_cells is 1024, #lstm_layers is 2
GAP in validate1 is 0.8298 at step 74796
53. distillchain_lstm_moe8
lstm and moe model as described in model #3, use ditill labels as label dependence in video level
youtube-8m-zhangteng.frame_level_models.LstmModel + youtube-8m-zhangteng.video_level_models.MoeDistillChainModel
#moe_mixtures is 8, #chain_relu_cells is 256, #lstm_cells is 1024, #lstm_layers is 2
GAP in validate1 is 0.8283 at step 49873
54. distillchain_lstm_multiscale_layer2_moe8
cnn-lstm and moe model as described in model #27, use ditill labels as label dependence in each cnn layer
youtube-8m-zhangteng.frame_level_models.LstmMultiscaleDitillChainModel
#moe_mixtures is 4, #chain_relu_cells is 256, #cnn_layers is 2, #cnn_cells is [256,256,512], #cnn_fields is [1,2,3] , #lstm_cells is 1024, #lstm_layers is 1
GAP in validate1 is 0.8313 at step 103725
55. distillchain_lstm_multiscale_layer4_moe8
cnn-lstm and moe model as described in model #27, use ditill labels as label dependence in each cnn layer
youtube-8m-zhangteng.frame_level_models.LstmMultiscaleDitillChainModel
#moe_mixtures is 4, #chain_relu_cells is 256, #cnn_layers is 4, #cnn_cells is [196,196,392], #cnn_fields is [1,2,3] , #lstm_cells is 1024, #lstm_layers is 1
GAP in validate1 is 0.8320 at step 102004
56. distillchain_lstmcnn_dcc
57. distillsplit_lstm_gate_moe8
lstm and moe model, use lstm cell described in model #7 in frame level, use the last 3716 ditill labels as label dependence in video level to redict the rest 1000 labels, concat the 3716 ditill labels and 1000 predicted labels as the final predictions
youtube-8m-zhangteng.frame_level_models.LstmGateModel + youtube-8m-zhangteng.video_level_models.MoeDistillSplitModel
#moe_mixtures is 8, #chain_relu_cells is 256, #lstm_cells is 1024, #lstm_layers is 1
GAP in validate1 is 0.8257 at step 130570
58. distillchain_v2_video_dcc
59. distillchain_v2_lstmattention8max
60. distillchain_v2_lstmcnn_dcc
61. distillchain_v2_lstm_gate_moe8
lstm and moe model as described in model #49, but on a new distillation training dataset
youtube-8m-zhangteng.frame_level_models.LstmGateModel + youtube-8m-zhangteng.video_level_models.MoeDistillChainModel
#moe_mixtures is 8, #chain_relu_cells is 256, #lstm_cells is 1024, #lstm_layers is 1
GAP in validate1 is 0.8273 at step 73400
62. distillchain_v2_lstm_glu2_moe8
lstm and moe model as described in model #49, but use lstm cell as described in model #8, and on a new distillation training dataset
youtube-8m-zhangteng.frame_level_models.LstmGlu2Model + youtube-8m-zhangteng.video_level_models.MoeDistillChainModel
#moe_mixtures is 8, #chain_relu_cells is 256, #lstm_cells is 1024, #lstm_layers is 1
GAP in validate1 is 0.8280 at step 72525
63. distillchain_v2_multilstm_dcc
64. distillchain_v2_multiscale_cnnlstm
65. distillchain_v2_lstmparalleloutput
66. distillchain_v2_boost_lstmparalleloutput
67. videochain/distillchain_v2_video_dcc_1
68. videochain/distillchain_v2_video_dcc_2
69. videochain/distillchain_v2_video_dcc_3
70. videochain/distillchain_v2_video_dcc_4
71. videochain/distillchain_v2_video_dcc_5
72. hybridchain/distillchain_v2_video_dcc
73. hybridchain/distillchain_v2_cnn_dcc
74. hybridchain/distillchain_v2_lstmparalleloutput
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/infer-attention_linear_model.sh
================================================
model=$1
conf=$2
checkpoint=$3
test_path=/Youtube-8M/model_predictions/test
test_data_patterns=""
for d in $(cat $conf); do
test_data_patterns="${test_path}/${d}/*.tfrecord${test_data_patterns:+,$test_data_patterns}"
done
echo "$test_data_patterns"
input_data_pattern="${test_path}/model_input/*.tfrecord"
#CUDA_VISIBLE_DEVICES=0 python inference.py \
python inference.py \
--model_checkpoint_path="../model/${model}/model.ckpt-${checkpoint}" \
--output_file="../model/${model}/predictions.${model}.csv" \
--model="AttentionLinearModel" \
--moe_num_mixtures=16 \
--batch_size=1024 \
--input_data_pattern="$input_data_pattern" \
--input_data_patterns="$test_data_patterns"
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/infer-attention_linmatrix_model.sh
================================================
model=$1
conf=$2
checkpoint=$3
test_path=/Youtube-8M/model_predictions/test
test_data_patterns=""
for d in $(cat $conf); do
test_data_patterns="${test_path}/${d}/*.tfrecord${test_data_patterns:+,$test_data_patterns}"
done
echo "$test_data_patterns"
input_data_pattern="${test_path}/model_input/*.tfrecord"
#CUDA_VISIBLE_DEVICES=0 python inference.py \
python inference.py \
--model_checkpoint_path="../model/${model}/model.ckpt-${checkpoint}" \
--output_file="../model/${model}/predictions.${model}.csv" \
--model="AttentionLinmatrixModel" \
--moe_num_mixtures=16 \
--attention_matrix_rank=4 \
--batch_size=1024 \
--input_data_pattern="$input_data_pattern" \
--input_data_patterns="$test_data_patterns"
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/infer-attention_matrix_model.sh
================================================
model=$1
conf=$2
checkpoint=$3
test_path=/Youtube-8M/model_predictions/test
test_data_patterns=""
for d in $(cat $conf); do
test_data_patterns="${test_path}/${d}/*.tfrecord${test_data_patterns:+,$test_data_patterns}"
done
echo "$test_data_patterns"
input_data_pattern="${test_path}/model_input/*.tfrecord"
#CUDA_VISIBLE_DEVICES=0 python inference.py \
python inference.py \
--model_checkpoint_path="../model/${model}/model.ckpt-${checkpoint}" \
--output_file="../model/${model}/predictions.${model}.csv" \
--model="AttentionMatrixModel" \
--moe_num_mixtures=4 \
--attention_matrix_rank=8 \
--batch_size=1024 \
--input_data_pattern="$input_data_pattern" \
--input_data_patterns="$test_data_patterns"
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/infer-attention_moe_matrix_model.sh
================================================
model=$1
conf=$2
checkpoint=$3
test_path=/Youtube-8M/model_predictions/test
test_data_patterns=""
for d in $(cat $conf); do
test_data_patterns="${test_path}/${d}/*.tfrecord${test_data_patterns:+,$test_data_patterns}"
done
echo "$test_data_patterns"
input_data_pattern="${test_path}/model_input/*.tfrecord"
#CUDA_VISIBLE_DEVICES=0 python inference.py \
python inference.py \
--model_checkpoint_path="../model/${model}/model.ckpt-${checkpoint}" \
--output_file="../model/${model}/predictions.${model}.csv" \
--model="AttentionMatrixModel" \
--moe_num_mixtures=4 \
--attention_relu_cells=256 \
--attention_matrix_rank=4 \
--batch_size=1024 \
--input_data_pattern="$input_data_pattern" \
--input_data_patterns="$test_data_patterns"
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/infer-attention_moe_model.sh
================================================
model=$1
conf=$2
checkpoint=$3
test_path=/Youtube-8M/model_predictions/test
test_data_patterns=""
for d in $(cat $conf); do
test_data_patterns="${test_path}/${d}/*.tfrecord${test_data_patterns:+,$test_data_patterns}"
done
echo "$test_data_patterns"
input_data_pattern="${test_path}/model_input/*.tfrecord"
#CUDA_VISIBLE_DEVICES=0 python inference.py \
python inference.py \
--model_checkpoint_path="../model/${model}/model.ckpt-${checkpoint}" \
--output_file="../model/${model}/predictions.${model}.csv" \
--model=AttentionMoeModel \
--moe_num_mixtures=8 \
--attention_relu_cells=128 \
--batch_size=1024 \
--input_data_pattern="$input_data_pattern" \
--input_data_patterns="$test_data_patterns"
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/infer-attention_rectified_linear_model.sh
================================================
model=$1
conf=$2
checkpoint=$3
test_path=/Youtube-8M/model_predictions/test
test_data_patterns=""
for d in $(cat $conf); do
test_data_patterns="${test_path}/${d}/*.tfrecord${test_data_patterns:+,$test_data_patterns}"
done
echo "$test_data_patterns"
input_data_pattern="${test_path}/model_input/*.tfrecord"
#CUDA_VISIBLE_DEVICES=0 python inference.py \
python inference.py \
--model_checkpoint_path="../model/${model}/model.ckpt-${checkpoint}" \
--output_file="../model/${model}/predictions.${model}.csv" \
--model="AttentionRectifiedLinearModel" \
--moe_num_mixtures=16 \
--batch_size=1024 \
--input_data_pattern="$input_data_pattern" \
--input_data_patterns="$test_data_patterns"
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/infer-linear_model.sh
================================================
model=$1
conf=$2
checkpoint=$3
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
test_path=/Youtube-8M/model_predictions/test
test_data_patterns=""
for d in $(cat $conf); do
test_data_patterns="${test_path}/${d}/*.tfrecord${test_data_patterns:+,$test_data_patterns}"
done
echo "$test_data_patterns"
CUDA_VISIBLE_DEVICES=${GPU_ID} python inference.py \
--model_checkpoint_path="../model/${model}/model.ckpt-$checkpoint" \
--output_file="../model/${model}/predictions.csv" \
--model="LinearRegressionModel" \
--input_data_patterns="$test_data_patterns"
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/infer-matrix_model.sh
================================================
model=$1
conf=$2
checkpoint=$3
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
test_path=/Youtube-8M/model_predictions/test
test_data_patterns=""
for d in $(cat $conf); do
test_data_patterns="${test_path}/${d}/*.tfrecord${test_data_patterns:+,$test_data_patterns}"
done
echo "$test_data_patterns"
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference.py \
--model_checkpoint_path="../model/${model}/model.ckpt-${checkpoint}" \
--output_file="../model/${model}/predictions.csv" \
--model="MatrixRegressionModel" \
--batch_size=1024 \
--input_data_patterns="$test_data_patterns"
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/infer-mean_model.sh
================================================
model=$1
conf=$2
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
test_path=/Youtube-8M/model_predictions/test
test_data_patterns=""
for d in $(cat $conf); do
test_data_patterns="${test_path}/${d}/*.tfrecord${test_data_patterns:+,$test_data_patterns}"
done
echo "$test_data_patterns"
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference.py \
--model_checkpoint_path="../model/${model}/model.ckpt-0" \
--output_file="../model/${model}/predictions.csv" \
--model="MeanModel" \
--input_data_patterns="$test_data_patterns"
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/infer-moe_model.sh
================================================
model=$1
conf=$2
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
test_path=/Youtube-8M/model_predictions/test
test_data_patterns=""
for d in $(cat $conf); do
test_data_patterns="${test_path}/${d}/*.tfrecord${test_data_patterns:+,$test_data_patterns}"
done
echo "$test_data_patterns"
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference.py \
--model_checkpoint_path="../model/${model}/model.ckpt-2203" \
--output_file="../model/${model}/predictions.${model}.csv" \
--model="MoeModel" \
--input_data_patterns="$test_data_patterns"
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/make-bagging-of-ensembles.sh
================================================
#!/bin/bash
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
# base_model or sub_model_1 or sub_model_2 or so on
model_type="$1"
model_name="boe_ensemble_matrix_model_no10"
MODEL_DIR="../model/${model_name}"
main_conf="ensemble_scripts/ensemble_no10.conf"
# generate vocab file
vocab_file="resources/validate.video_id.vocab"
default_freq_file="resources/validate.video_id.freq"
if [ ! -f $vocab_file ]; then
cd resources
wget http://us.data.yt8m.org/1/ground_truth_labels/validate_labels.csv
echo "OOV" > validate.video_id.vocab
cat validate_labels.csv | cut -d ',' -f 1 >> validate.video_id.vocab
cd ..
fi
vocab_checksum=$(md5sum $vocab_file | cut -d ' ' -f 1)
if [ "$vocab_checksum" == "4a0c959434bb540f926d17a492161ca8" ]; then
echo $vocab_file is valid
else
echo $vocab_file is corrupted
exit 1
fi
if [ ! -f $default_freq_file ]; then
cat $vocab_file | awk '{print 1}' > $default_freq_file
fi
rm ${MODEL_DIR}/ensemble.conf
mkdir $MODEL_DIR
for i in {1..8}; do
# sub model
model_type="sub_model_${i}"
sub_model_dir="${MODEL_DIR}/${model_type}"
sub_conf="${sub_model_dir}/ensemble.conf"
sub_freq="${sub_model_dir}/validate.video_id.freq"
if [ ! -d $sub_model_dir ]; then
mkdir -p $sub_model_dir
# generate freq file
python training_utils/sample_freq.py \
--video_id_file="$vocab_file" \
--output_freq_file="$sub_freq"
# generate conf file
python training_utils/sample_conf.py \
--main_conf_file="$main_conf" \
--sub_conf_file="$sub_conf"
# train data patterns for ensemble training
train_path=/Youtube-8M/model_predictions/ensemble_train
sub_train_data_patterns=""
for d in $(cat $sub_conf); do
sub_train_data_patterns="${train_path}/${d}/*.tfrecord${sub_train_data_patterns:+,$sub_train_data_patterns}"
done
echo "$sub_train_data_patterns"
# train N models with re-weighted samples
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="$sub_model_dir" \
--train_data_patterns="$sub_train_data_patterns" \
--reweight=True \
--sample_vocab_file="$vocab_file" \
--sample_freq_file="$sub_freq" \
--model=MatrixRegressionModel \
--keep_checkpoint_every_n_hours=0.1 \
--batch_size=1024 \
--num_epochs=2
# inference-pre-ensemble
for part in test ensemble_validate ensemble_train; do
output_dir="/Youtube-8M/model_predictions/${part}/${model_name}/${model_type}"
if [ ! -d $output_dir ]; then
# test data patterns
test_path=/Youtube-8M/model_predictions/${part}
sub_test_data_patterns=""
for d in $(cat $sub_conf); do
sub_test_data_patterns="${test_path}/${d}/*.tfrecord${sub_test_data_patterns:+,$sub_test_data_patterns}"
done
echo "$sub_test_data_patterns"
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-pre-ensemble.py \
--output_dir="$output_dir" \
--train_dir="$sub_model_dir" \
--input_data_patterns="$sub_test_data_patterns" \
--model="MatrixRegressionModel" \
--batch_size=1024 \
--file_size=4096
fi
done
fi
echo "${model_name}/${model_type}" >> ${MODEL_DIR}/ensemble.conf
done
# on ensemble server
bash ensemble_scripts/train-attention_matrix_model.sh ${model_name}/ensemble_attention_matrix_model ${MODEL_DIR}/ensemble.conf
bash ensemble_scripts/eval-attention_matrix_model.sh ${model_name}/ensemble_attention_matrix_model ${MODEL_DIR}/ensemble.conf
# bash ensemble_scripts/infer-mean_model.sh ${model_name}/ensemble_mean_model ${MODEL_DIR}/ensemble.conf
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/make-virtual-groups.sh
================================================
#!/bin/bash
group_dir="../model/virtual_grouping"
for file in $(ls $group_dir); do
if [[ $file =~ ^virtual_group ]] && [[ $file =~ conf$ ]]; then
group_name="${file:0:-5}"
model="virtual_grouping/${group_name}"
model_dir="${group_dir}/${group_name}"
if [ ! -d $model_dir ]; then
conf="${group_dir}/${group_name}.conf"
echo "training $model ..."
bash ensemble_scripts/train-matrix_model.sh $model $conf
bash ensemble_scripts/eval-matrix_model.sh $model $conf
fi
for part in ensemble_train ensemble_validate test; do
output_dir="/Youtube-8M/model_predictions/${part}/${model}"
if [ ! -d $output_dir ]; then
bash ensemble_scripts/auto-preensemble-matrix_model.sh $model $conf $part
fi
done
fi
done
model_dir="${group_dir}/ensemble_matrix_model"
if [ ! -d $model_dir ]; then
model="virtual_grouping/ensemble_matrix_model"
conf="${group_dir}/ensemble.conf"
echo "training $model ..."
bash ensemble_scripts/train-matrix_model.sh $model $conf
bash ensemble_scripts/eval-matrix_model.sh $model $conf
fi
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/preensemble-attention_matrix_model.sh
================================================
model=$1
conf=$2
part=$3
checkpoint=$4
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
test_path=/Youtube-8M/model_predictions/${part}
test_data_patterns=""
for d in $(cat $conf); do
test_data_patterns="${test_path}/${d}/*.tfrecord${test_data_patterns:+,$test_data_patterns}"
done
echo "$test_data_patterns"
input_data_pattern="${test_path}/model_input/*.tfrecord"
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/${model}" \
--model_checkpoint_path="../model/${model}/model.ckpt-${checkpoint}" \
--input_data_patterns="$test_data_patterns" \
--input_data_pattern="$input_data_pattern" \
--model="AttentionMatrixModel" \
--moe_num_mixtures=4 \
--attention_matrix_rank=8 \
--batch_size=1024 \
--file_size=4096
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/preensemble-matrix_model.sh
================================================
model=$1
conf=$2
part=$3
checkpoint=$4
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
test_path=/Youtube-8M/model_predictions/${part}
test_data_patterns=""
for d in $(cat $conf); do
test_data_patterns="${test_path}/${d}/*.tfrecord${test_data_patterns:+,$test_data_patterns}"
done
echo "$test_data_patterns"
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/${model}" \
--model_checkpoint_path="../model/${model}/model.ckpt-${checkpoint}" \
--input_data_patterns="$test_data_patterns" \
--model="MatrixRegressionModel" \
--batch_size=1024 \
--file_size=4096
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/preensemble-mean_model.sh
================================================
model=$1
conf=$2
part=$3
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
test_path=/Youtube-8M/model_predictions/${part}
test_data_patterns=""
for d in $(cat $conf); do
test_data_patterns="${test_path}/${d}/*.tfrecord${test_data_patterns:+,$test_data_patterns}"
done
echo "$test_data_patterns"
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/${model}" \
--model_checkpoint_path="../model/${model}/model.ckpt-0" \
--input_data_patterns="$test_data_patterns" \
--model="MeanModel" \
--batch_size=1024 \
--file_size=4096
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/train-attention_linear_model.sh
================================================
model=$1
conf=$2
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
train_path=/Youtube-8M/model_predictions/ensemble_train
train_data_patterns=""
for d in $(cat $conf); do
train_data_patterns="${train_path}/${d}/*.tfrecord${train_data_patterns:+,$train_data_patterns}"
done
echo $train_data_patterns
input_data_pattern="${train_path}/model_input/*.tfrecord"
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="../model/${model}" \
--train_data_patterns="$train_data_patterns" \
--input_data_pattern="$input_data_pattern" \
--model=AttentionLinearModel \
--base_learning_rate=0.01 \
--moe_num_mixtures=16 \
--keep_checkpoint_every_n_hours=0.1 \
--batch_size=1024 \
--num_epochs=4
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/train-attention_linmatrix_model.sh
================================================
model=$1
conf=$2
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
train_path=/Youtube-8M/model_predictions/ensemble_train
train_data_patterns=""
for d in $(cat $conf); do
train_data_patterns="${train_path}/${d}/*.tfrecord${train_data_patterns:+,$train_data_patterns}"
done
echo $train_data_patterns
input_data_pattern="${train_path}/model_input/*.tfrecord"
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="../model/${model}" \
--train_data_patterns="$train_data_patterns" \
--input_data_pattern="$input_data_pattern" \
--model=AttentionLinmatrixModel \
--base_learning_rate=0.004 \
--moe_num_mixtures=16 \
--attention_matrix_rank=4 \
--keep_checkpoint_every_n_hours=0.1 \
--batch_size=1024 \
--num_epochs=4
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/train-attention_matrix_model.sh
================================================
model=$1
conf=$2
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
train_path=/Youtube-8M/model_predictions/ensemble_train
train_data_patterns=""
for d in $(cat $conf); do
train_data_patterns="${train_path}/${d}/*.tfrecord${train_data_patterns:+,$train_data_patterns}"
done
echo $train_data_patterns
input_data_pattern="${train_path}/model_input/*.tfrecord"
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="../model/${model}" \
--train_data_patterns="$train_data_patterns" \
--input_data_pattern="$input_data_pattern" \
--model=AttentionMatrixModel \
--base_learning_rate=0.004 \
--moe_num_mixtures=4 \
--attention_matrix_rank=8 \
--keep_checkpoint_every_n_hours=0.1 \
--batch_size=1024 \
--num_epochs=4
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/train-attention_moe_matrix_model.sh
================================================
model=$1
conf=$2
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
train_path=/Youtube-8M/model_predictions/ensemble_train
train_data_patterns=""
for d in $(cat $conf); do
train_data_patterns="${train_path}/${d}/*.tfrecord${train_data_patterns:+,$train_data_patterns}"
done
echo $train_data_patterns
input_data_pattern="${train_path}/model_input/*.tfrecord"
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="../model/${model}" \
--train_data_patterns="$train_data_patterns" \
--input_data_pattern="$input_data_pattern" \
--model=AttentionMatrixModel \
--base_learning_rate=0.01 \
--moe_num_mixtures=4 \
--attention_relu_cells=256 \
--attention_matrix_rank=4 \
--keep_checkpoint_every_n_hours=0.1 \
--batch_size=1024 \
--num_epochs=4
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/train-attention_moe_model.sh
================================================
model=$1
conf=$2
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
train_path=/Youtube-8M/model_predictions/ensemble_train
train_data_patterns=""
for d in $(cat $conf); do
train_data_patterns="${train_path}/${d}/*.tfrecord${train_data_patterns:+,$train_data_patterns}"
done
echo $train_data_patterns
input_data_pattern="${train_path}/model_input/*.tfrecord"
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="../model/${model}" \
--train_data_patterns="$train_data_patterns" \
--input_data_pattern="$input_data_pattern" \
--model=AttentionMoeModel \
--moe_num_mixtures=8 \
--attention_relu_cells=128 \
--base_learning_rate=0.01 \
--keep_checkpoint_every_n_hours=0.1 \
--batch_size=1024 \
--num_epochs=4
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/train-attention_rectified_linear_model.sh
================================================
model=$1
conf=$2
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
train_path=/Youtube-8M/model_predictions/ensemble_train
train_data_patterns=""
for d in $(cat $conf); do
train_data_patterns="${train_path}/${d}/*.tfrecord${train_data_patterns:+,$train_data_patterns}"
done
echo $train_data_patterns
input_data_pattern="${train_path}/model_input/*.tfrecord"
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="../model/${model}" \
--train_data_patterns="$train_data_patterns" \
--input_data_pattern="$input_data_pattern" \
--model=AttentionRectifiedLinearModel \
--base_learning_rate=0.01 \
--moe_num_mixtures=16 \
--keep_checkpoint_every_n_hours=0.1 \
--batch_size=1024 \
--num_epochs=4
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/train-deep_combine_chain_model.sh
================================================
model=$1
conf=$2
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
train_path=/Youtube-8M/model_predictions/ensemble_train
train_data_patterns=""
labels=""
for d in $(cat $conf); do
train_data_patterns="${train_path}/${d}/*.tfrecord${train_data_patterns:+,$train_data_patterns}"
labels="label,${labels:+,$labels}"
done
echo $train_data_patterns
input_data_pattern="${train_path}/model_input/*.tfrecord"
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="../model/${model}" \
--train_data_patterns="$train_data_patterns" \
--input_data_pattern="$input_data_pattern" \
--model="DeepCombineChainModel" \
--deep_chain_relu_cells=128 \
--moe_num_mixtures=2 \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_loss_percent=0.2 \
--support_type="label" \
--base_learning_rate=0.001 \
--keep_checkpoint_every_n_hours=0.1 \
--batch_size=128 \
--num_epochs=4
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/train-input_moe_model.sh
================================================
model=$1
conf=$2
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
train_path=/Youtube-8M/model_predictions/ensemble_train
train_data_patterns=""
for d in $(cat $conf); do
train_data_patterns="${train_path}/${d}/*.tfrecord${train_data_patterns:+,$train_data_patterns}"
done
echo $train_data_patterns
input_data_pattern="${train_path}/model_input/*.tfrecord"
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="../model/${model}" \
--train_data_patterns="$train_data_patterns" \
--input_data_pattern="$input_data_pattern" \
--model=InputMoeModel \
--keep_checkpoint_every_n_hours=0.1 \
--batch_size=1024 \
--num_epochs=5
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/train-linear_model.sh
================================================
model=$1
conf=$2
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
train_path=/Youtube-8M/model_predictions/ensemble_train
train_data_patterns=""
for d in $(cat $conf); do
train_data_patterns="${train_path}/${d}/*.tfrecord${train_data_patterns:+,$train_data_patterns}"
done
echo $train_data_patterns
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="../model/${model}" \
--train_data_patterns="$train_data_patterns" \
--model=LinearRegressionModel \
--keep_checkpoint_every_n_hours=0.1 \
--batch_size=1024 \
--num_epochs=1
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/train-matrix_model.sh
================================================
model=$1
conf=$2
postfix=$3
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
train_path=/Youtube-8M/model_predictions${postfix}/ensemble_train
train_data_patterns=""
for d in $(cat $conf); do
train_data_patterns="${train_path}/${d}/*.tfrecord${train_data_patterns:+,$train_data_patterns}"
done
echo "$train_data_patterns"
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="../model/${model}" \
--train_data_patterns="$train_data_patterns" \
--model=MatrixRegressionModel \
--keep_checkpoint_every_n_hours=0.1 \
--batch_size=1024 \
--num_epochs=2
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/train-matrix_model_lr.sh
================================================
model=$1
conf=$2
learn_rate=$3
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
train_path=/Youtube-8M/model_predictions/ensemble_train
train_data_patterns=""
for d in $(cat $conf); do
train_data_patterns="${train_path}/${d}/*.tfrecord${train_data_patterns:+,$train_data_patterns}"
done
echo "$train_data_patterns"
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="../model/${model}" \
--train_data_patterns="$train_data_patterns" \
--model=MatrixRegressionModel \
--base_learning_rate=$learn_rate \
--keep_checkpoint_every_n_hours=0.1 \
--batch_size=1024 \
--num_epochs=2
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/train-mean_model.sh
================================================
model=$1
conf=$2
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
train_path=/Youtube-8M/model_predictions/ensemble_train
train_data_patterns=""
for d in $(cat $conf); do
train_data_patterns="${train_path}/${d}/*.tfrecord${train_data_patterns:+,$train_data_patterns}"
done
echo $train_data_patterns
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="../model/${model}" \
--train_data_patterns="$train_data_patterns" \
--model=MeanModel \
--training=False \
--num_epochs=1
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/train-moe_model.sh
================================================
model=$1
conf=$2
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
train_path=/Youtube-8M/model_predictions/ensemble_train
train_data_patterns=""
for d in $(cat $conf); do
train_data_patterns="${train_path}/${d}/*.tfrecord${train_data_patterns:+,$train_data_patterns}"
done
echo $train_data_patterns
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="../model/${model}" \
--train_data_patterns="$train_data_patterns" \
--model=MoeModel \
--keep_checkpoint_every_n_hours=0.1 \
--batch_size=1024 \
--num_epochs=5
================================================
FILE: youtube-8m-ensemble/ensemble_scripts/train-nonunit_matrix_model.sh
================================================
model=$1
conf=$2
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
train_path=/Youtube-8M/model_predictions/ensemble_train
train_data_patterns=""
for d in $(cat $conf); do
train_data_patterns="${train_path}/${d}/*.tfrecord${train_data_patterns:+,$train_data_patterns}"
done
echo $train_data_patterns
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="../model/${model}" \
--train_data_patterns="$train_data_patterns" \
--model=NonunitMatrixRegressionModel \
--keep_checkpoint_every_n_hours=0.1 \
--batch_size=1024 \
--num_epochs=5
================================================
FILE: youtube-8m-ensemble/eval.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for evaluating Tensorflow models on the YouTube-8M dataset."""
import time
import eval_util
import losses
import ensemble_level_models
import readers
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import utils
FLAGS = flags.FLAGS
if __name__ == "__main__":
# Dataset flags.
flags.DEFINE_string("model_checkpoint_path", "",
"The file to load the model files from. ")
flags.DEFINE_string("train_dir", "/tmp/yt8m/",
"The directory to write the result in. ")
flags.DEFINE_string(
"eval_data_patterns", "",
"File globs defining the evaluation dataset in tensorflow.SequenceExample format.")
flags.DEFINE_string(
"input_data_pattern", None,
"File globs for original model input.")
flags.DEFINE_string("feature_names", "predictions", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "4716", "Length of the feature vectors.")
# Model flags.
flags.DEFINE_string(
"model", "LinearRegressionModel",
"Which architecture to use for the model.")
flags.DEFINE_integer("batch_size", 256,
"How many examples to process per batch.")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Loss computed on validation data")
# Other flags.
flags.DEFINE_boolean("run_once", True, "Whether to run eval only once.")
flags.DEFINE_boolean("echo_gap", False, "Whether to echo GAP at the end.")
flags.DEFINE_integer("top_k", 20, "How many predictions to output per video.")
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def get_input_evaluation_tensors(reader,
data_pattern,
batch_size=256):
logging.info("Using batch size of " + str(batch_size) + " for evaluation.")
with tf.name_scope("eval_input"):
files = gfile.Glob(data_pattern)
if not files:
print data_pattern, files
raise IOError("Unable to find the evaluation files.")
logging.info("number of evaluation files: " + str(len(files)))
files.sort()
filename_queue = tf.train.string_input_producer(
files, shuffle=False, num_epochs=1)
eval_data = reader.prepare_reader(filename_queue)
return tf.train.batch(
eval_data,
batch_size=batch_size,
capacity=3 * batch_size,
allow_smaller_final_batch=True,
enqueue_many=True)
def build_graph(all_readers,
input_reader,
all_eval_data_patterns,
input_data_pattern,
model,
label_loss_fn,
batch_size=256):
"""Creates the Tensorflow graph for evaluation.
Args:
all_readers: The data file reader. It should inherit from BaseReader.
model: The core model (e.g. logistic or neural net). It should inherit
from BaseModel.
all_eval_data_patterns: glob path to the evaluation data files.
label_loss_fn: What kind of loss to apply to the model. It should inherit
from BaseLoss.
batch_size: How many examples to process at a time.
"""
global_step = tf.Variable(0, trainable=False, name="global_step")
model_input_raw_tensors = []
labels_batch_tensor = None
video_id_batch = None
for reader, data_pattern in zip(all_readers, all_eval_data_patterns):
unused_video_id, model_input_raw, labels_batch, unused_num_frames = (
get_input_evaluation_tensors(
reader,
data_pattern,
batch_size=batch_size))
if labels_batch_tensor is None:
labels_batch_tensor = labels_batch
if video_id_batch is None:
video_id_batch = unused_video_id
model_input_raw_tensors.append(tf.expand_dims(model_input_raw, axis=2))
original_input = None
if input_data_pattern is not None:
unused_video_id, original_input, unused_labels_batch, unused_num_frames = (
get_input_evaluation_tensors(
input_reader,
input_data_pattern,
batch_size=batch_size))
model_input = tf.concat(model_input_raw_tensors, axis=2)
labels_batch = labels_batch_tensor
with tf.name_scope("model"):
result = model.create_model(model_input,
labels=labels_batch,
vocab_size=reader.num_classes,
original_input=original_input,
is_training=False)
predictions = result["predictions"]
tf.summary.histogram("model_activations", predictions)
if "loss" in result.keys():
label_loss = result["loss"]
else:
label_loss = label_loss_fn.calculate_loss(predictions, labels_batch)
tf.add_to_collection("global_step", global_step)
tf.add_to_collection("loss", label_loss)
tf.add_to_collection("predictions", predictions)
tf.add_to_collection("input_batch", model_input)
tf.add_to_collection("video_id_batch", video_id_batch)
tf.add_to_collection("labels", tf.cast(labels_batch, tf.float32))
tf.add_to_collection("summary_op", tf.summary.merge_all())
def evaluation_loop(video_id_batch, prediction_batch, label_batch, loss,
summary_op, saver, summary_writer, evl_metrics,
last_global_step_val):
"""Run the evaluation loop once.
Args:
video_id_batch: a tensor of video ids mini-batch.
prediction_batch: a tensor of predictions mini-batch.
label_batch: a tensor of label_batch mini-batch.
loss: a tensor of loss for the examples in the mini-batch.
summary_op: a tensor which runs the tensorboard summary operations.
saver: a tensorflow saver to restore the model.
summary_writer: a tensorflow summary_writer
evl_metrics: an EvaluationMetrics object.
last_global_step_val: the global step used in the previous evaluation.
Returns:
The global_step used in the latest model.
"""
global_step_val = -1
with tf.Session() as sess:
checkpoint = FLAGS.model_checkpoint_path
if checkpoint:
logging.info("Loading checkpoint for eval: " + checkpoint)
# Restores from checkpoint
saver.restore(sess, checkpoint)
# Assuming model_checkpoint_path looks something like:
# /my-favorite-path/yt8m_train/model.ckpt-0, extract global_step from it.
global_step_val = checkpoint.split("/")[-1].split("-")[-1]
else:
logging.info("No checkpoint file found.")
return global_step_val
sess.run([tf.local_variables_initializer()])
# Start the queue runners.
fetches = [video_id_batch, prediction_batch, label_batch, loss, summary_op]
coord = tf.train.Coordinator()
try:
threads = []
for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
threads.extend(qr.create_threads(
sess, coord=coord, daemon=True,
start=True))
logging.info("enter eval_once loop global_step_val = %s. ",
global_step_val)
evl_metrics.clear()
examples_processed = 0
while not coord.should_stop():
batch_start_time = time.time()
_, predictions_val, labels_val, loss_val, summary_val = sess.run(fetches)
seconds_per_batch = time.time() - batch_start_time
example_per_second = labels_val.shape[0] / seconds_per_batch
examples_processed += labels_val.shape[0]
iteration_info_dict = evl_metrics.accumulate(predictions_val,
labels_val, loss_val)
iteration_info_dict["examples_per_second"] = example_per_second
iterinfo = utils.AddGlobalStepSummary(
summary_writer,
global_step_val,
iteration_info_dict,
summary_scope="Eval")
logging.info("examples_processed: %d | %s", examples_processed,
iterinfo)
except tf.errors.OutOfRangeError as e:
logging.info(
"Done with batched inference. Now calculating global performance "
"metrics.")
# calculate the metrics for the entire epoch
epoch_info_dict = evl_metrics.get()
epoch_info_dict["epoch_id"] = global_step_val
summary_writer.add_summary(summary_val, global_step_val)
epochinfo = utils.AddEpochSummary(
summary_writer,
global_step_val,
epoch_info_dict,
summary_scope="Eval")
if FLAGS.echo_gap:
print "GAP =", epoch_info_dict["gap"]
logging.info(epochinfo)
evl_metrics.clear()
except Exception as e: # pylint: disable=broad-except
logging.info("Unexpected exception: " + str(e))
coord.request_stop(e)
coord.request_stop()
coord.join(threads, stop_grace_period_secs=10)
return global_step_val
def evaluate():
tf.set_random_seed(0) # for reproducibility
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
# prepare a reader for each single model prediction result
all_readers = []
all_patterns = FLAGS.eval_data_patterns
all_patterns = map(lambda x: x.strip(), all_patterns.strip().strip(",").split(","))
for i in xrange(len(all_patterns)):
reader = readers.EnsembleReader(
feature_names=feature_names, feature_sizes=feature_sizes)
all_readers.append(reader)
input_reader = None
input_data_pattern = None
if FLAGS.input_data_pattern is not None:
input_reader = readers.EnsembleReader(
feature_names=["input"], feature_sizes=[1024+128])
input_data_pattern = FLAGS.input_data_pattern
# find the model
model = find_class_by_name(FLAGS.model, [ensemble_level_models])()
label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])()
if FLAGS.eval_data_patterns is "":
raise IOError("'eval_data_patterns' was not specified. " +
"Nothing to evaluate.")
build_graph(
all_readers=all_readers,
input_reader=input_reader,
all_eval_data_patterns=all_patterns,
input_data_pattern=input_data_pattern,
model=model,
label_loss_fn=label_loss_fn,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
video_id_batch = tf.get_collection("video_id_batch")[0]
prediction_batch = tf.get_collection("predictions")[0]
label_batch = tf.get_collection("labels")[0]
loss = tf.get_collection("loss")[0]
summary_op = tf.get_collection("summary_op")[0]
saver = tf.train.Saver(tf.global_variables())
summary_writer = tf.summary.FileWriter(
FLAGS.train_dir, graph=tf.get_default_graph())
evl_metrics = eval_util.EvaluationMetrics(FLAGS.num_classes, FLAGS.top_k)
last_global_step_val = -1
last_global_step_val = evaluation_loop(video_id_batch, prediction_batch,
label_batch, loss, summary_op,
saver, summary_writer, evl_metrics,
last_global_step_val)
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
evaluate()
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-ensemble/eval_util.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Provides functions to help with evaluating models."""
import datetime
import numpy
from tensorflow.python.platform import gfile
import mean_average_precision_calculator as map_calculator
import average_precision_calculator as ap_calculator
def flatten(l):
""" Merges a list of lists into a single list. """
return [item for sublist in l for item in sublist]
def calculate_hit_at_one(predictions, actuals):
"""Performs a local (numpy) calculation of the hit at one.
Args:
predictions: Matrix containing the outputs of the model.
Dimensions are 'batch' x 'num_classes'.
actuals: Matrix containing the ground truth labels.
Dimensions are 'batch' x 'num_classes'.
Returns:
float: The average hit at one across the entire batch.
"""
top_prediction = numpy.argmax(predictions, 1)
hits = actuals[numpy.arange(actuals.shape[0]), top_prediction]
return numpy.average(hits)
def calculate_recall_at_n(predictions, actuals, n):
"""Performs a local (numpy) calculation of the recall@n
Args:
predictions: Matrix containing the outputs of the model.
Dimensions are 'batch' x 'num_classes'.
actuals: Matrix containing the ground truth labels.
Dimensions are 'batch' x 'num_classes'.
n: scalar of n
Returns:
float: The recall at n across the entire batch.
"""
aggregated_recall = 0.0
num_videos = actuals.shape[0]
for row in numpy.arange(num_videos):
num_labels = int(numpy.sum(actuals[row]))
top_indices = numpy.argpartition(predictions[row],
-n)[-n:]
item_recall = 0.0
for label_index in top_indices:
if predictions[row][label_index] > 0:
item_recall += actuals[row][label_index]
item_recall /= num_labels
aggregated_recall += item_recall
aggregated_recall /= num_videos
return aggregated_recall
def calculate_precision_at_equal_recall_rate(predictions, actuals):
"""Performs a local (numpy) calculation of the PERR.
Args:
predictions: Matrix containing the outputs of the model.
Dimensions are 'batch' x 'num_classes'.
actuals: Matrix containing the ground truth labels.
Dimensions are 'batch' x 'num_classes'.
Returns:
float: The average precision at equal recall rate across the entire batch.
"""
aggregated_precision = 0.0
num_videos = actuals.shape[0]
for row in numpy.arange(num_videos):
num_labels = int(numpy.sum(actuals[row]))
top_indices = numpy.argpartition(predictions[row],
-num_labels)[-num_labels:]
item_precision = 0.0
for label_index in top_indices:
if predictions[row][label_index] > 0:
item_precision += actuals[row][label_index]
item_precision /= top_indices.size
aggregated_precision += item_precision
aggregated_precision /= num_videos
return aggregated_precision
def calculate_gap(predictions, actuals, top_k=20):
"""Performs a local (numpy) calculation of the global average precision.
Only the top_k predictions are taken for each of the videos.
Args:
predictions: Matrix containing the outputs of the model.
Dimensions are 'batch' x 'num_classes'.
actuals: Matrix containing the ground truth labels.
Dimensions are 'batch' x 'num_classes'.
top_k: How many predictions to use per video.
Returns:
float: The global average precision.
"""
gap_calculator = ap_calculator.AveragePrecisionCalculator()
sparse_predictions, sparse_labels, num_positives = top_k_by_class(predictions, actuals, top_k)
gap_calculator.accumulate(flatten(sparse_predictions), flatten(sparse_labels), sum(num_positives))
return gap_calculator.peek_ap_at_n()
def top_k_by_class(predictions, labels, k=20):
"""Extracts the top k predictions for each video, sorted by class.
Args:
predictions: A numpy matrix containing the outputs of the model.
Dimensions are 'batch' x 'num_classes'.
k: the top k non-zero entries to preserve in each prediction.
Returns:
A tuple (predictions,labels, true_positives). 'predictions' and 'labels'
are lists of lists of floats. 'true_positives' is a list of scalars. The
length of the lists are equal to the number of classes. The entries in the
predictions variable are probability predictions, and
the corresponding entries in the labels variable are the ground truth for
those predictions. The entries in 'true_positives' are the number of true
positives for each class in the ground truth.
Raises:
ValueError: An error occurred when the k is not a positive integer.
"""
if k <= 0:
raise ValueError("k must be a positive integer.")
k = min(k, predictions.shape[1])
num_classes = predictions.shape[1]
prediction_triplets= []
for video_index in range(predictions.shape[0]):
prediction_triplets.extend(top_k_triplets(predictions[video_index],labels[video_index], k))
out_predictions = [[] for v in range(num_classes)]
out_labels = [[] for v in range(num_classes)]
for triplet in prediction_triplets:
out_predictions[triplet[0]].append(triplet[1])
out_labels[triplet[0]].append(triplet[2])
out_true_positives = [numpy.sum(labels[:,i]) for i in range(num_classes)]
return out_predictions, out_labels, out_true_positives
def top_k_triplets(predictions, labels, k=20):
"""Get the top_k for a 1-d numpy array. Returns a sparse list of tuples in
(prediction, class) format"""
m = len(predictions)
k = min(k, m)
indices = numpy.argpartition(predictions, -k)[-k:]
return [(index, predictions[index], labels[index]) for index in indices]
class EvaluationMetrics(object):
"""A class to store the evaluation metrics."""
def __init__(self, num_class, top_k):
"""Construct an EvaluationMetrics object to store the evaluation metrics.
Args:
num_class: A positive integer specifying the number of classes.
top_k: A positive integer specifying how many predictions are considered per video.
Raises:
ValueError: An error occurred when MeanAveragePrecisionCalculator cannot
not be constructed.
"""
self.sum_hit_at_one = 0.0
self.sum_perr = 0.0
self.sum_loss = 0.0
self.map_calculator = map_calculator.MeanAveragePrecisionCalculator(num_class)
self.global_ap_calculator = ap_calculator.AveragePrecisionCalculator()
self.top_k = top_k
self.num_examples = 0
def accumulate(self, predictions, labels, loss):
"""Accumulate the metrics calculated locally for this mini-batch.
Args:
predictions: A numpy matrix containing the outputs of the model.
Dimensions are 'batch' x 'num_classes'.
labels: A numpy matrix containing the ground truth labels.
Dimensions are 'batch' x 'num_classes'.
loss: A numpy array containing the loss for each sample.
Returns:
dictionary: A dictionary storing the metrics for the mini-batch.
Raises:
ValueError: An error occurred when the shape of predictions and actuals
does not match.
"""
batch_size = labels.shape[0]
mean_hit_at_one = calculate_hit_at_one(predictions, labels)
mean_perr = calculate_precision_at_equal_recall_rate(predictions, labels)
mean_loss = numpy.mean(loss)
# Take the top 20 predictions.
sparse_predictions, sparse_labels, num_positives = top_k_by_class(predictions, labels, self.top_k)
self.map_calculator.accumulate(sparse_predictions, sparse_labels, num_positives)
self.global_ap_calculator.accumulate(flatten(sparse_predictions), flatten(sparse_labels), sum(num_positives))
self.num_examples += batch_size
self.sum_hit_at_one += mean_hit_at_one * batch_size
self.sum_perr += mean_perr * batch_size
self.sum_loss += mean_loss * batch_size
return {"hit_at_one": mean_hit_at_one, "perr": mean_perr, "loss": mean_loss}
def get(self):
"""Calculate the evaluation metrics for the whole epoch.
Raises:
ValueError: If no examples were accumulated.
Returns:
dictionary: a dictionary storing the evaluation metrics for the epoch. The
dictionary has the fields: avg_hit_at_one, avg_perr, avg_loss, and
aps (default nan).
"""
if self.num_examples <= 0:
raise ValueError("total_sample must be positive.")
avg_hit_at_one = self.sum_hit_at_one / self.num_examples
avg_perr = self.sum_perr / self.num_examples
avg_loss = self.sum_loss / self.num_examples
aps = self.map_calculator.peek_map_at_n()
gap = self.global_ap_calculator.peek_ap_at_n()
epoch_info_dict = {}
return {"avg_hit_at_one": avg_hit_at_one, "avg_perr": avg_perr,
"avg_loss": avg_loss, "aps": aps, "gap": gap}
def clear(self):
"""Clear the evaluation metrics and reset the EvaluationMetrics object."""
self.sum_hit_at_one = 0.0
self.sum_perr = 0.0
self.sum_loss = 0.0
self.map_calculator.clear()
self.global_ap_calculator.clear()
self.num_examples = 0
================================================
FILE: youtube-8m-ensemble/feature_transform.py
================================================
from tensorflow import flags
flags.DEFINE_string("feature_transformer", "DefaultTransformer",
"how to preprocess feature, defaults to identical, which means no transform")
flags.DEFINE_string("engineer_types", "identical,avg,std,diff",
"how to preprocess feature, defaults to identical, which means no transform")
flags.DEFINE_integer("time_resolution", 8,
"how many frames to mean pooling at a time")
from all_feature_transform import *
================================================
FILE: youtube-8m-ensemble/inference-combine-tfrecords-frame.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for combine model output and model input into one set of files."""
import os
import sys
import time
import numpy
import numpy as np
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import utils
import eval_util
import losses
import readers
import ensemble_level_models
FLAGS = flags.FLAGS
if __name__ == '__main__':
flags.DEFINE_string("output_dir", "",
"The file to save the predictions to.")
flags.DEFINE_string(
"input_data_pattern", "",
"File globs defining the input dataset in tensorflow.SequenceExample format.")
flags.DEFINE_string(
"prediction_data_pattern", "",
"File globs defining the output dataset in tensorflow.SequenceExample format.")
flags.DEFINE_string("input_feature_names", "mean_rgb,mean_audio", "Name of the feature "
"to use for training.")
flags.DEFINE_string("input_feature_sizes", "1024,128", "Length of the feature vectors.")
flags.DEFINE_string("prediction_feature_names", "predictions", "Name of the feature "
"to use for training.")
flags.DEFINE_string("prediction_feature_sizes", "4716", "Length of the feature vectors.")
flags.DEFINE_integer("batch_size", 256,
"How many examples to process per batch.")
flags.DEFINE_integer("file_size", 4096,
"Number of frames per batch for DBoF.")
flags.DEFINE_integer("file_num_mod", None,
"file_num % 3 == file_num_mod will be output.")
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def get_input_data_tensors(reader,
data_pattern,
batch_size=256):
logging.info("Using batch size of " + str(batch_size) + " for evaluation.")
with tf.name_scope("eval_input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find the evaluation files.")
logging.info("number of evaluation files: " + str(len(files)))
files.sort()
filename_queue = tf.train.string_input_producer(
files, shuffle=False, num_epochs=1)
eval_data = reader.prepare_reader(filename_queue)
return tf.train.batch(
eval_data,
batch_size=batch_size,
capacity=4 * batch_size,
allow_smaller_final_batch=True,
enqueue_many=True)
def build_graph(input_reader, input_data_pattern,
prediction_reader, prediction_data_pattern,
batch_size=256):
"""Creates the Tensorflow graph for evaluation.
Args:
all_readers: The data file reader. It should inherit from BaseReader.
model: The core model (e.g. logistic or neural net). It should inherit
from BaseModel.
all_data_patterns: glob path to the evaluation data files.
label_loss_fn: What kind of loss to apply to the model. It should inherit
from BaseLoss.
batch_size: How many examples to process at a time.
"""
video_ids_batch, model_rgbs_batch, model_audios_batch, labels_batch, num_frames_batch = (
get_input_data_tensors(
input_reader,
input_data_pattern,
batch_size=batch_size))
video_ids_batch2, model_predictions_batch, labels_batch2, unused_num_frames2 = (
get_input_data_tensors(
prediction_reader,
prediction_data_pattern,
batch_size=batch_size))
tf.add_to_collection("video_ids_batch", video_ids_batch)
tf.add_to_collection("labels_batch", tf.cast(labels_batch, tf.float32))
tf.add_to_collection("rgbs_batch", model_rgbs_batch)
tf.add_to_collection("audios_batch", model_audios_batch)
tf.add_to_collection("predictions_batch", model_predictions_batch)
tf.add_to_collection("num_frames_batch", num_frames_batch)
def inference_loop(video_ids_batch, labels_batch, rgbs_batch, audios_batch, predictions_batch, num_frames_batch,
output_dir, batch_size):
with tf.Session() as sess:
sess.run([tf.local_variables_initializer()])
# Start the queue runners.
fetches = [video_ids_batch, labels_batch, rgbs_batch, audios_batch, predictions_batch, num_frames_batch]
coord = tf.train.Coordinator()
start_time = time.time()
video_ids = []
video_labels = []
video_rgbs = []
video_audios = []
video_predictions = []
video_num_frames = []
filenum = 0
num_examples_processed = 0
total_num_examples_processed = 0
directory = FLAGS.output_dir
if not os.path.exists(directory):
os.makedirs(directory)
try:
threads = []
for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
threads.extend(qr.create_threads(
sess, coord=coord, daemon=True,
start=True))
while not coord.should_stop():
ids_val = None
ids_val, labels_val, rgbs_val, audios_val, predictions_val, num_frames_val = sess.run(fetches)
if filenum % 3 == FLAGS.file_num_mod or FLAGS.file_num_mod is None:
video_ids.append(ids_val)
video_labels.append(labels_val)
video_rgbs.append(rgbs_val)
video_audios.append(audios_val)
video_predictions.append(predictions_val)
video_num_frames.append(num_frames_val)
num_examples_processed += len(ids_val)
ids_shape = ids_val.shape[0]
rgbs_shape = rgbs_val.shape[0]
audios_shape = audios_val.shape[0]
predictions_shape = predictions_val.shape[0]
assert ids_shape == rgbs_shape == predictions_shape, "tensor ids(%d), rgbs(%d) and predictions(%d) should have equal rows" % (ids_shape, rgbs_shape, predictions_shape)
ids_val = None
now = time.time()
logging.info("num examples processed: " + str(num_examples_processed) + " elapsed seconds: " + "{0:.2f}".format(now-start_time))
if num_examples_processed >= FLAGS.file_size:
assert num_examples_processed==FLAGS.file_size, "num_examples_processed should be equal to %d"%FLAGS.file_size
if filenum % 3 == FLAGS.file_num_mod or FLAGS.file_num_mod is None:
video_ids = np.concatenate(video_ids, axis=0)
video_labels = np.concatenate(video_labels, axis=0)
video_rgbs = np.concatenate(video_rgbs, axis=0)
video_audios = np.concatenate(video_audios, axis=0)
video_predictions = np.concatenate(video_predictions, axis=0)
video_num_frames = np.concatenate(video_num_frames, axis=0)
write_to_record(video_ids, video_labels, video_rgbs, video_audios, video_predictions, video_num_frames, filenum, num_examples_processed)
video_ids = []
video_labels = []
video_rgbs = []
video_audios = []
video_predictions = []
video_num_frames = []
filenum += 1
total_num_examples_processed += num_examples_processed
now = time.time()
logging.info(str(num_examples_processed) + " examples written to file elapsed seconds: " + "{0:.2f}".format(now-start_time))
num_examples_processed = 0
except tf.errors.OutOfRangeError as e:
if ids_val is not None:
video_ids.append(ids_val)
video_labels.append(labels_val)
video_rgbs.append(rgbs_val)
video_audios.append(audios_val)
video_predictions.append(predictions_val)
video_num_frames.append(num_frames_val)
num_examples_processed += len(ids_val)
if 0 < num_examples_processed <= FLAGS.file_size:
if filenum % 3 == FLAGS.file_num_mod or FLAGS.file_num_mod is None:
video_ids = np.concatenate(video_ids, axis=0)
video_labels = np.concatenate(video_labels, axis=0)
video_rgbs = np.concatenate(video_rgbs, axis=0)
video_audios = np.concatenate(video_audios, axis=0)
video_predictions = np.concatenate(video_predictions, axis=0)
video_num_frames = np.concatenate(video_num_frames, axis=0)
write_to_record(video_ids, video_labels, video_rgbs, video_audios, video_predictions, video_num_frames, filenum, num_examples_processed)
total_num_examples_processed += num_examples_processed
now = time.time()
logging.info("num examples processed: " + str(num_examples_processed) + " elapsed seconds: " + "{0:.2f}".format(now-start_time))
num_examples_processed = 0
logging.info("Done with inference. %d samples was written to %s" % (total_num_examples_processed, FLAGS.output_dir))
except Exception as e: # pylint: disable=broad-except
logging.info("Unexpected exception: " + str(e))
finally:
coord.request_stop()
coord.join(threads, stop_grace_period_secs=10)
def write_to_record(video_ids, video_labels, video_rgbs, video_audios, video_predictions, video_num_frames, filenum, num_examples_processed):
writer = tf.python_io.TFRecordWriter(FLAGS.output_dir + '/' + 'predictions-%04d.tfrecord' % filenum)
for i in range(num_examples_processed):
video_id = video_ids[i]
video_label = np.nonzero(video_labels[i,:])[0]
video_rgb = video_rgbs[i,:]
video_audio = video_audios[i,:]
video_prediction = video_predictions[i,:]
video_num_frame = video_num_frames[i]
example = get_output_feature(video_id, video_label, video_rgb, video_audio, video_prediction, video_num_frame)
serialized = example.SerializeToString()
writer.write(serialized)
writer.close()
def get_output_feature(video_id, video_label, video_rgb, video_audio, video_prediction, video_num_frame):
_bytes_feature_list = lambda x: tf.train.Feature(bytes_list=tf.train.BytesList(value=[x]))
example = tf.train.SequenceExample(
context = tf.train.Features(feature={
"video_id": tf.train.Feature(bytes_list=tf.train.BytesList(value=[video_id])),
"labels": tf.train.Feature(int64_list=tf.train.Int64List(value=video_label)),
"predictions": tf.train.Feature(float_list=tf.train.FloatList(value=video_prediction))
}),
feature_lists = tf.train.FeatureLists(feature_list={
"rgb": tf.train.FeatureList(feature=map(_bytes_feature_list, video_rgb[:video_num_frame])),
"audio": tf.train.FeatureList(feature=map(_bytes_feature_list, video_audio[:video_num_frame])),
})
)
return example
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
if FLAGS.input_data_pattern is "":
raise IOError("'input_data_pattern' was not specified. " +
"Nothing to evaluate.")
if FLAGS.prediction_data_pattern is "":
raise IOError("'prediction_data_pattern' was not specified. " +
"Nothing to evaluate.")
# convert feature_names and feature_sizes to lists of values
input_feature_names, input_feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.input_feature_names, FLAGS.input_feature_sizes)
input_reader = readers.EnsembleFrameReader(
feature_names=input_feature_names,
feature_sizes=input_feature_sizes)
prediction_feature_names, prediction_feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.prediction_feature_names, FLAGS.prediction_feature_sizes)
prediction_reader = readers.EnsembleReader(
feature_names=prediction_feature_names,
feature_sizes=prediction_feature_sizes)
build_graph(
input_reader=input_reader,
prediction_reader=prediction_reader,
input_data_pattern=FLAGS.input_data_pattern,
prediction_data_pattern=FLAGS.prediction_data_pattern,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
video_ids_batch = tf.get_collection("video_ids_batch")[0]
labels_batch = tf.get_collection("labels_batch")[0]
rgbs_batch = tf.get_collection("rgbs_batch")[0]
audios_batch = tf.get_collection("audios_batch")[0]
predictions_batch = tf.get_collection("predictions_batch")[0]
num_frames_batch = tf.get_collection("num_frames_batch")[0]
inference_loop(video_ids_batch, labels_batch, rgbs_batch, audios_batch, predictions_batch, num_frames_batch,
FLAGS.output_dir, FLAGS.batch_size)
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-ensemble/inference-combine-tfrecords-video.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for combine model output and model input into one set of files."""
import os
import time
import numpy
import numpy as np
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import utils
import eval_util
import losses
import readers
import ensemble_level_models
FLAGS = flags.FLAGS
if __name__ == '__main__':
flags.DEFINE_string("output_dir", "",
"The file to save the predictions to.")
flags.DEFINE_string(
"input_data_pattern", "",
"File globs defining the input dataset in tensorflow.SequenceExample format.")
flags.DEFINE_string(
"prediction_data_pattern", "",
"File globs defining the output dataset in tensorflow.SequenceExample format.")
flags.DEFINE_string("input_feature_names", "mean_rgb,mean_audio", "Name of the feature "
"to use for training.")
flags.DEFINE_string("input_feature_sizes", "1024,128", "Length of the feature vectors.")
flags.DEFINE_string("prediction_feature_names", "predictions", "Name of the feature "
"to use for training.")
flags.DEFINE_string("prediction_feature_sizes", "4716", "Length of the feature vectors.")
flags.DEFINE_integer("batch_size", 256,
"How many examples to process per batch.")
flags.DEFINE_integer("file_size", 4096,
"Number of frames per batch for DBoF.")
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def get_input_data_tensors(reader,
data_pattern,
batch_size=256):
logging.info("Using batch size of " + str(batch_size) + " for evaluation.")
with tf.name_scope("eval_input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find the evaluation files.")
logging.info("number of evaluation files: " + str(len(files)))
files.sort()
filename_queue = tf.train.string_input_producer(
files, shuffle=False, num_epochs=1)
eval_data = reader.prepare_reader(filename_queue)
return tf.train.batch(
eval_data,
batch_size=batch_size,
capacity=batch_size,
allow_smaller_final_batch=True,
enqueue_many=True)
def build_graph(input_reader, input_data_pattern,
prediction_reader, prediction_data_pattern,
batch_size=256):
"""Creates the Tensorflow graph for evaluation.
Args:
all_readers: The data file reader. It should inherit from BaseReader.
model: The core model (e.g. logistic or neural net). It should inherit
from BaseModel.
all_data_patterns: glob path to the evaluation data files.
label_loss_fn: What kind of loss to apply to the model. It should inherit
from BaseLoss.
batch_size: How many examples to process at a time.
"""
video_ids_batch, model_inputs_batch, labels_batch, unused_num_frames = (
get_input_data_tensors(
input_reader,
input_data_pattern,
batch_size=batch_size))
video_ids_batch2, model_predictions_batch, labels_batch2, unused_num_frames2 = (
get_input_data_tensors(
prediction_reader,
prediction_data_pattern,
batch_size=batch_size))
video_ids_equal = tf.reduce_mean(tf.cast(tf.equal(video_ids_batch, video_ids_batch2), tf.float32))
labels_equal = tf.reduce_mean(tf.reduce_sum(tf.cast(tf.equal(labels_batch, labels_batch2), tf.float32), axis=1))
tf.add_to_collection("video_ids_equal", video_ids_equal)
tf.add_to_collection("labels_equal", labels_equal)
tf.add_to_collection("video_ids_batch", video_ids_batch)
tf.add_to_collection("labels_batch", tf.cast(labels_batch, tf.float32))
tf.add_to_collection("inputs_batch", model_inputs_batch)
tf.add_to_collection("predictions_batch", model_predictions_batch)
def inference_loop(video_ids_batch, labels_batch, inputs_batch, predictions_batch, video_ids_equal, labels_equal,
output_dir, batch_size):
with tf.Session() as sess:
sess.run([tf.local_variables_initializer()])
# Start the queue runners.
fetches = [video_ids_batch, labels_batch, inputs_batch, predictions_batch, video_ids_equal, labels_equal]
coord = tf.train.Coordinator()
start_time = time.time()
video_ids = []
video_labels = []
video_inputs = []
video_predictions = []
filenum = 0
num_examples_processed = 0
total_num_examples_processed = 0
directory = FLAGS.output_dir
if not os.path.exists(directory):
os.makedirs(directory)
else:
raise IOError("Output path exists! path='" + directory + "'")
try:
threads = []
for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
threads.extend(qr.create_threads(
sess, coord=coord, daemon=True,
start=True))
while not coord.should_stop():
ids_val = None
ids_val, labels_val, inputs_val, predictions_val, ids_equal_val, labels_equal_val = sess.run(fetches)
print "ids equal = %f" % (ids_equal_val)
print "labels equal = %f" % (labels_equal_val)
video_ids.append(ids_val)
video_labels.append(labels_val)
video_inputs.append(inputs_val)
video_predictions.append(predictions_val)
num_examples_processed += len(ids_val)
ids_shape = ids_val.shape[0]
inputs_shape = inputs_val.shape[0]
predictions_shape = predictions_val.shape[0]
assert ids_shape == inputs_shape == predictions_shape, "tensor ids(%d), inputs(%d) and predictions(%d) should have equal rows" % (ids_shape, inputs_shape, predictions_shape)
ids_val = None
if num_examples_processed >= FLAGS.file_size:
assert num_examples_processed==FLAGS.file_size, "num_examples_processed should be equal to %d"%FLAGS.file_size
video_ids = np.concatenate(video_ids, axis=0)
video_labels = np.concatenate(video_labels, axis=0)
video_inputs = np.concatenate(video_inputs, axis=0)
video_predictions = np.concatenate(video_predictions, axis=0)
write_to_record(video_ids, video_labels, video_inputs, video_predictions, filenum, num_examples_processed)
video_ids = []
video_labels = []
video_inputs = []
video_predictions = []
filenum += 1
total_num_examples_processed += num_examples_processed
now = time.time()
logging.info("num examples processed: " + str(num_examples_processed) + " elapsed seconds: " + "{0:.2f}".format(now-start_time))
num_examples_processed = 0
except tf.errors.OutOfRangeError as e:
if ids_val is not None:
video_ids.append(ids_val)
video_labels.append(labels_val)
video_inputs.append(inputs_val)
video_predictions.append(predictions_val)
num_examples_processed += len(ids_val)
if 0 < num_examples_processed <= FLAGS.file_size:
video_ids = np.concatenate(video_ids, axis=0)
video_labels = np.concatenate(video_labels, axis=0)
video_inputs = np.concatenate(video_inputs, axis=0)
video_predictions = np.concatenate(video_predictions, axis=0)
write_to_record(video_ids, video_labels, video_inputs, video_predictions, filenum, num_examples_processed)
total_num_examples_processed += num_examples_processed
now = time.time()
logging.info("num examples processed: " + str(num_examples_processed) + " elapsed seconds: " + "{0:.2f}".format(now-start_time))
num_examples_processed = 0
logging.info("Done with inference. %d samples was written to %s" % (total_num_examples_processed, FLAGS.output_dir))
except Exception as e: # pylint: disable=broad-except
logging.info("Unexpected exception: " + str(e))
finally:
coord.request_stop()
coord.join(threads, stop_grace_period_secs=10)
def write_to_record(video_ids, video_labels, video_inputs, video_predictions, filenum, num_examples_processed):
writer = tf.python_io.TFRecordWriter(FLAGS.output_dir + '/' + 'predictions-%04d.tfrecord' % filenum)
for i in range(num_examples_processed):
video_id = video_ids[i]
video_label = np.nonzero(video_labels[i,:])[0]
video_input = video_inputs[i,:]
video_prediction = video_predictions[i,:]
example = get_output_feature(video_id, video_label, video_input, video_prediction)
serialized = example.SerializeToString()
writer.write(serialized)
writer.close()
def get_output_feature(video_id, video_label, video_input, video_prediction):
feature_maps = {'video_id': tf.train.Feature(bytes_list=tf.train.BytesList(value=[video_id])),
'labels': tf.train.Feature(int64_list=tf.train.Int64List(value=video_label))}
input_feature_names = FLAGS.input_feature_names.split(",")
input_feature_sizes = map(int, FLAGS.input_feature_sizes.split(","))
input_feature_start = 0
for i in range(len(input_feature_names)):
feature_maps[input_feature_names[i]] = tf.train.Feature(
float_list=tf.train.FloatList(
value=video_input[
input_feature_start :
input_feature_start + input_feature_sizes[i]]))
input_feature_start += input_feature_sizes[i]
prediction_feature_names = FLAGS.prediction_feature_names.split(",")
prediction_feature_sizes = map(int, FLAGS.prediction_feature_sizes.split(","))
prediction_feature_start = 0
for i in range(len(prediction_feature_names)):
feature_maps[prediction_feature_names[i]] = tf.train.Feature(
float_list=tf.train.FloatList(
value=video_prediction[
prediction_feature_start :
prediction_feature_start + prediction_feature_sizes[i]]))
prediction_feature_start += prediction_feature_sizes[i]
example = tf.train.Example(features=tf.train.Features(feature=feature_maps))
return example
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
if FLAGS.input_data_pattern is "":
raise IOError("'input_data_pattern' was not specified. " +
"Nothing to evaluate.")
if FLAGS.prediction_data_pattern is "":
raise IOError("'prediction_data_pattern' was not specified. " +
"Nothing to evaluate.")
# convert feature_names and feature_sizes to lists of values
input_feature_names, input_feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.input_feature_names, FLAGS.input_feature_sizes)
input_reader = readers.EnsembleReader(
feature_names=input_feature_names,
feature_sizes=input_feature_sizes)
prediction_feature_names, prediction_feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.prediction_feature_names, FLAGS.prediction_feature_sizes)
prediction_reader = readers.EnsembleReader(
feature_names=prediction_feature_names,
feature_sizes=prediction_feature_sizes)
build_graph(
input_reader=input_reader,
prediction_reader=prediction_reader,
input_data_pattern=FLAGS.input_data_pattern,
prediction_data_pattern=FLAGS.prediction_data_pattern,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
video_ids_equal = tf.get_collection("video_ids_equal")[0]
labels_equal = tf.get_collection("labels_equal")[0]
video_ids_batch = tf.get_collection("video_ids_batch")[0]
labels_batch = tf.get_collection("labels_batch")[0]
inputs_batch = tf.get_collection("inputs_batch")[0]
predictions_batch = tf.get_collection("predictions_batch")[0]
inference_loop(video_ids_batch, labels_batch, inputs_batch, predictions_batch, video_ids_equal, labels_equal,
FLAGS.output_dir, FLAGS.batch_size)
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-ensemble/inference-pre-ensemble.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for generating predictions over a set of videos."""
import os
import time
import numpy
import numpy as np
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import utils
import eval_util
import losses
import readers
import ensemble_level_models
FLAGS = flags.FLAGS
if __name__ == '__main__':
flags.DEFINE_string("model_checkpoint_path", None,
"The file path to load the model from.")
flags.DEFINE_string("train_dir", "",
"The directory to load the model from.")
flags.DEFINE_string("output_dir", "",
"The file to save the predictions to.")
flags.DEFINE_string(
"input_data_patterns", "",
"File globs defining the evaluation dataset in tensorflow.SequenceExample format.")
flags.DEFINE_string(
"input_data_pattern", None,
"File globs for original model input.")
flags.DEFINE_string("feature_names", "predictions", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "4716", "Length of the feature vectors.")
# Model flags.
flags.DEFINE_string(
"model", "MeanModel",
"Which architecture to use for the model.")
flags.DEFINE_integer("batch_size", 256,
"How many examples to process per batch.")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Loss computed on validation data")
flags.DEFINE_integer("file_size", 4096,
"Number of frames per batch for DBoF.")
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def get_input_data_tensors(reader,
data_pattern,
batch_size=256):
logging.info("Using batch size of " + str(batch_size) + " for evaluation.")
with tf.name_scope("eval_input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find the evaluation files.")
logging.info("number of evaluation files: " + str(len(files)))
files.sort()
filename_queue = tf.train.string_input_producer(
files, shuffle=False, num_epochs=1)
eval_data = reader.prepare_reader(filename_queue)
return tf.train.batch(
eval_data,
batch_size=batch_size,
capacity=4 * batch_size,
allow_smaller_final_batch=True,
enqueue_many=True)
def build_graph(all_readers,
all_data_patterns,
input_reader,
input_data_pattern,
model,
label_loss_fn,
batch_size=256):
"""Creates the Tensorflow graph for evaluation.
Args:
all_readers: The data file reader. It should inherit from BaseReader.
model: The core model (e.g. logistic or neural net). It should inherit
from BaseModel.
all_data_patterns: glob path to the evaluation data files.
label_loss_fn: What kind of loss to apply to the model. It should inherit
from BaseLoss.
batch_size: How many examples to process at a time.
"""
global_step = tf.Variable(0, trainable=False, name="global_step")
model_input_raw_tensors = []
labels_batch_tensor = None
video_id_batch = None
for reader, data_pattern in zip(all_readers, all_data_patterns):
unused_video_id, model_input_raw, labels_batch, unused_num_frames = (
get_input_data_tensors(
reader,
data_pattern,
batch_size=batch_size))
if labels_batch_tensor is None:
labels_batch_tensor = labels_batch
if video_id_batch is None:
video_id_batch = unused_video_id
model_input_raw_tensors.append(tf.expand_dims(model_input_raw, axis=2))
original_input = None
if input_data_pattern is not None:
unused_video_id, original_input, unused_labels_batch, unused_num_frames = (
get_input_data_tensors(
input_reader,
input_data_pattern,
batch_size=batch_size))
model_input = tf.concat(model_input_raw_tensors, axis=2)
labels_batch = labels_batch_tensor
with tf.name_scope("model"):
result = model.create_model(model_input,
labels=labels_batch,
vocab_size=reader.num_classes,
original_input=original_input,
is_training=False)
predictions = result["predictions"]
if "loss" in result.keys():
label_loss = result["loss"]
else:
label_loss = label_loss_fn.calculate_loss(predictions, labels_batch)
tf.add_to_collection("global_step", global_step)
tf.add_to_collection("loss", label_loss)
tf.add_to_collection("predictions", predictions)
tf.add_to_collection("input_batch", model_input)
tf.add_to_collection("video_id_batch", video_id_batch)
tf.add_to_collection("labels", tf.cast(labels_batch, tf.float32))
def inference_loop(video_id_batch, prediction_batch,
label_batch, saver,
output_dir, batch_size):
with tf.Session() as sess:
checkpoint = FLAGS.model_checkpoint_path
if checkpoint is None:
checkpoint = tf.train.latest_checkpoint(FLAGS.train_dir)
if checkpoint:
logging.info("Loading checkpoint for eval: " + checkpoint)
saver.restore(sess, checkpoint)
global_step_val = checkpoint.split("/")[-1].split("-")[-1]
else:
logging.info("No checkpoint file found.")
return global_step_val
sess.run([tf.local_variables_initializer()])
# Start the queue runners.
fetches = [video_id_batch, prediction_batch, label_batch]
coord = tf.train.Coordinator()
start_time = time.time()
video_ids = []
video_labels = []
video_features = []
filenum = 0
num_examples_processed = 0
total_num_examples_processed = 0
directory = FLAGS.output_dir
if not os.path.exists(directory):
os.makedirs(directory)
else:
raise IOError("Output path exists! path='" + directory + "'")
try:
threads = []
for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
threads.extend(qr.create_threads(
sess, coord=coord, daemon=True,
start=True))
logging.info("enter eval_once loop global_step_val = %s. ",
global_step_val)
while not coord.should_stop():
ids_val, predictions_val, labels_val = None, None, None
ids_val, predictions_val, labels_val = sess.run(fetches)
video_ids.append(ids_val)
video_labels.append(labels_val)
video_features.append(predictions_val)
num_examples_processed += len(ids_val)
ids_val, predictions_val, labels_val = None, None, None
if num_examples_processed >= FLAGS.file_size:
assert num_examples_processed==FLAGS.file_size, "num_examples_processed should be equal to %d"%FLAGS.file_size
video_ids = np.concatenate(video_ids, axis=0)
video_labels = np.concatenate(video_labels, axis=0)
video_features = np.concatenate(video_features, axis=0)
write_to_record(video_ids, video_labels, video_features, filenum, num_examples_processed)
video_ids = []
video_labels = []
video_features = []
filenum += 1
total_num_examples_processed += num_examples_processed
now = time.time()
logging.info("num examples processed: " + str(num_examples_processed) + " elapsed seconds: " + "{0:.2f}".format(now-start_time))
num_examples_processed = 0
except tf.errors.OutOfRangeError as e:
if ids_val is not None:
video_ids.append(ids_val)
video_labels.append(labels_val)
video_features.append(predictions_val)
num_examples_processed += len(ids_val)
if 0 < num_examples_processed <= FLAGS.file_size:
video_ids = np.concatenate(video_ids, axis=0)
video_labels = np.concatenate(video_labels, axis=0)
video_features = np.concatenate(video_features, axis=0)
write_to_record(video_ids, video_labels, video_features, filenum, num_examples_processed)
total_num_examples_processed += num_examples_processed
now = time.time()
logging.info("num examples processed: " + str(num_examples_processed) + " elapsed seconds: " + "{0:.2f}".format(now-start_time))
num_examples_processed = 0
logging.info("Done with inference. %d samples was written to %s" % (total_num_examples_processed, FLAGS.output_dir))
except Exception as e: # pylint: disable=broad-except
logging.info("Unexpected exception: " + str(e))
finally:
coord.request_stop()
coord.join(threads, stop_grace_period_secs=10)
def write_to_record(video_ids, video_labels, video_features, filenum, num_examples_processed):
writer = tf.python_io.TFRecordWriter(FLAGS.output_dir + '/' + 'predictions-%04d.tfrecord' % filenum)
for i in range(num_examples_processed):
video_id = video_ids[i]
video_label = np.nonzero(video_labels[i,:])[0]
example = get_output_feature(video_id, video_label, [video_features[i,:]], ['predictions'])
serialized = example.SerializeToString()
writer.write(serialized)
writer.close()
def get_output_feature(video_id, video_label, video_feature, feature_names):
feature_maps = {'video_id': tf.train.Feature(bytes_list=tf.train.BytesList(value=[video_id])),
'labels': tf.train.Feature(int64_list=tf.train.Int64List(value=video_label))}
for feature_index in range(len(feature_names)):
feature_maps[feature_names[feature_index]] = tf.train.Feature(
float_list=tf.train.FloatList(value=video_feature[feature_index]))
example = tf.train.Example(features=tf.train.Features(feature=feature_maps))
return example
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
# prepare a reader for each single model prediction result
all_readers = []
all_patterns = FLAGS.input_data_patterns
all_patterns = map(lambda x: x.strip(), all_patterns.strip().strip(",").split(","))
for i in xrange(len(all_patterns)):
reader = readers.EnsembleReader(
feature_names=feature_names, feature_sizes=feature_sizes)
all_readers.append(reader)
input_reader = None
input_data_pattern = None
if FLAGS.input_data_pattern is not None:
input_reader = readers.EnsembleReader(
feature_names=["input"], feature_sizes=[1024+128])
input_data_pattern = FLAGS.input_data_pattern
model = find_class_by_name(FLAGS.model, [ensemble_level_models])()
label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])()
if FLAGS.input_data_patterns is "":
raise IOError("'input_data_patterns' was not specified. " +
"Nothing to evaluate.")
build_graph(
all_readers=all_readers,
input_reader=input_reader,
all_data_patterns=all_patterns,
input_data_pattern=input_data_pattern,
model=model,
label_loss_fn=label_loss_fn,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
video_id_batch = tf.get_collection("video_id_batch")[0]
prediction_batch = tf.get_collection("predictions")[0]
label_batch = tf.get_collection("labels")[0]
saver = tf.train.Saver(tf.global_variables())
inference_loop(video_id_batch, prediction_batch,
label_batch, saver,
FLAGS.output_dir, FLAGS.batch_size)
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-ensemble/inference.py
================================================
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for evaluating Tensorflow models on the YouTube-8M dataset."""
import time
import numpy
import eval_util
import losses
import ensemble_level_models
import readers
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import utils
FLAGS = flags.FLAGS
if __name__ == "__main__":
# Dataset flags.
flags.DEFINE_string("model_checkpoint_path", None,
"The file to load the model files from. ")
flags.DEFINE_string("output_file", "",
"The file to save the predictions to.")
flags.DEFINE_string(
"input_data_patterns", "",
"File globs defining the evaluation dataset in tensorflow.SequenceExample format.")
flags.DEFINE_string(
"input_data_pattern", None,
"File globs for original model input.")
flags.DEFINE_string("feature_names", "predictions", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "4716", "Length of the feature vectors.")
# Model flags.
flags.DEFINE_string(
"model", "LogisticModel",
"Which architecture to use for the model.")
flags.DEFINE_integer("batch_size", 256,
"How many examples to process per batch.")
# Other flags.
flags.DEFINE_boolean("run_once", True, "Whether to run eval only once.")
flags.DEFINE_integer("top_k", 20, "How many predictions to output per video.")
def format_lines(video_ids, predictions, top_k):
batch_size = len(video_ids)
for video_index in range(batch_size):
top_indices = numpy.argpartition(predictions[video_index], -top_k)[-top_k:]
line = [(class_index, predictions[video_index][class_index])
for class_index in top_indices]
line = sorted(line, key=lambda p: -p[1])
yield video_ids[video_index].decode('utf-8') + "," + " ".join("%i %f" % pair
for pair in line) + "\n"
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def get_input_data_tensors(reader,
data_pattern,
batch_size=256):
logging.info("Using batch size of " + str(batch_size) + " for evaluation.")
with tf.name_scope("eval_input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find the evaluation files.")
logging.info("number of evaluation files: " + str(len(files)))
files.sort()
filename_queue = tf.train.string_input_producer(
files, shuffle=False, num_epochs=1)
eval_data = reader.prepare_reader(filename_queue)
return tf.train.batch(
eval_data,
batch_size=batch_size,
capacity=3 * batch_size,
allow_smaller_final_batch=True,
enqueue_many=True)
def build_graph(all_readers,
all_data_patterns,
input_reader,
input_data_pattern,
model,
batch_size=256):
"""Creates the Tensorflow graph for evaluation.
Args:
all_readers: The data file reader. It should inherit from BaseReader.
model: The core model (e.g. logistic or neural net). It should inherit
from BaseModel.
all_data_patterns: glob path to the evaluation data files.
batch_size: How many examples to process at a time.
"""
global_step = tf.Variable(0, trainable=False, name="global_step")
model_input_raw_tensors = []
labels_batch_tensor = None
video_id_batch = None
for reader, data_pattern in zip(all_readers, all_data_patterns):
unused_video_id, model_input_raw, labels_batch, unused_num_frames = (
get_input_data_tensors(
reader,
data_pattern,
batch_size=batch_size))
if labels_batch_tensor is None:
labels_batch_tensor = labels_batch
if video_id_batch is None:
video_id_batch = unused_video_id
model_input_raw_tensors.append(tf.expand_dims(model_input_raw, axis=2))
original_input = None
if input_data_pattern is not None:
unused_video_id, original_input, unused_labels_batch, unused_num_frames = (
get_input_data_tensors(
input_reader,
input_data_pattern,
batch_size=batch_size))
model_input = tf.concat(model_input_raw_tensors, axis=2)
labels_batch = labels_batch_tensor
with tf.name_scope("model"):
result = model.create_model(model_input,
labels=labels_batch,
vocab_size=reader.num_classes,
original_input=original_input,
is_training=False)
predictions = result["predictions"]
tf.add_to_collection("global_step", global_step)
tf.add_to_collection("predictions", predictions)
tf.add_to_collection("input_batch", model_input)
tf.add_to_collection("video_id_batch", video_id_batch)
tf.add_to_collection("labels", tf.cast(labels_batch, tf.float32))
def inference_loop(video_id_batch, prediction_batch, label_batch,
saver, out_file_location):
top_k = FLAGS.top_k
with tf.Session() as sess, gfile.Open(out_file_location, "w+") as out_file:
checkpoint = FLAGS.model_checkpoint_path
if checkpoint:
logging.info("Loading checkpoint for eval: " + checkpoint)
saver.restore(sess, checkpoint)
global_step_val = checkpoint.split("/")[-1].split("-")[-1]
else:
logging.info("No checkpoint file found.")
return global_step_val
sess.run([tf.local_variables_initializer()])
# Start the queue runners.
fetches = [video_id_batch, prediction_batch]
coord = tf.train.Coordinator()
try:
threads = []
for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
threads.extend(qr.create_threads(
sess, coord=coord, daemon=True,
start=True))
logging.info("enter eval_once loop global_step_val = %s. ",
global_step_val)
num_examples_processed = 0
start_time = time.time()
out_file.write("VideoId,LabelConfidencePairs\n")
while not coord.should_stop():
batch_start_time = time.time()
video_id_val, predictions_val = sess.run(fetches)
now = time.time()
num_examples_processed += len(video_id_val)
logging.info("num examples processed: " + str(num_examples_processed) + " elapsed seconds: " + "{0:.2f}".format(now-start_time))
for line in format_lines(video_id_val, predictions_val, top_k):
out_file.write(line)
out_file.flush()
except tf.errors.OutOfRangeError as e:
logging.info('Done with inference. The output file was written to ' + out_file_location)
except Exception as e: # pylint: disable=broad-except
logging.info("Unexpected exception: " + str(e))
coord.request_stop(e)
coord.request_stop()
coord.join(threads, stop_grace_period_secs=10)
def inference():
tf.set_random_seed(0) # for reproducibility
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
# prepare a reader for each single model prediction result
all_readers = []
all_patterns = FLAGS.input_data_patterns
all_patterns = map(lambda x: x.strip(), all_patterns.strip().strip(",").split(","))
for i in xrange(len(all_patterns)):
reader = readers.EnsembleReader(
feature_names=feature_names, feature_sizes=feature_sizes)
all_readers.append(reader)
input_reader = None
input_data_pattern = None
if FLAGS.input_data_pattern is not None:
input_reader = readers.EnsembleReader(
feature_names=["input"], feature_sizes=[1024+128])
input_data_pattern = FLAGS.input_data_pattern
model = find_class_by_name(FLAGS.model, [ensemble_level_models])()
if FLAGS.input_data_patterns is "":
raise IOError("'input_data_patterns' was not specified. " +
"Nothing to evaluate.")
build_graph(
all_readers=all_readers,
all_data_patterns=all_patterns,
input_reader=input_reader,
input_data_pattern=input_data_pattern,
model=model,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
video_id_batch = tf.get_collection("video_id_batch")[0]
prediction_batch = tf.get_collection("predictions")[0]
label_batch = tf.get_collection("labels")[0]
saver = tf.train.Saver(tf.global_variables())
inference_loop(video_id_batch, prediction_batch, label_batch,
saver, FLAGS.output_file)
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
inference()
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-ensemble/losses.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Provides definitions for non-regularized training or test losses."""
import numpy as np
import tensorflow as tf
from tensorflow import flags
FLAGS = flags.FLAGS
flags.DEFINE_float("false_negative_punishment", 1.0,
"punishment constant to 1 classified to 0")
flags.DEFINE_float("false_positive_punishment", 1.0,
"punishment constant to 0 classified to 1")
flags.DEFINE_integer("num_classes", 4716,
"number of classes")
flags.DEFINE_float("support_loss_percent", 0.1,
"the part that support loss (in multi-task scenario) take in the whole loss function.")
flags.DEFINE_string("support_type", "vertical",
"type of support label, vertical or frequent or vertical,frequent.")
flags.DEFINE_integer("num_supports", 25, "Number of total support categories.")
flags.DEFINE_integer("num_verticals", 25, "Number of total vertical categories.")
flags.DEFINE_integer("num_frequents", 200, "Number of total frequent categories.")
flags.DEFINE_string("vertical_file", "resources/vertical.tsv", "Location of label-vertical mapping file.")
flags.DEFINE_float("batch_agreement", 0.1,
"the batch_agreement parameter")
flags.DEFINE_bool("label_smoothing", False,
"whether do label smoothing")
flags.DEFINE_float("label_smoothing_epsilon", 0.1,
"whether do label smoothing")
def smoothing(labels):
print "label smoothing for", labels
epsilon = FLAGS.label_smoothing_epsilon
float_labels = tf.cast(labels, tf.float32)
num_labels = tf.reduce_sum(float_labels, axis=1, keep_dims=True)
K = float_labels.get_shape().as_list()[1]
prior = num_labels / K
smooth_labels = float_labels * (1.0 - epsilon) + prior * epsilon
return smooth_labels
class BaseLoss(object):
"""Inherit from this class when implementing new losses."""
def calculate_loss(self, unused_predictions, unused_labels, **unused_params):
"""Calculates the average loss of the examples in a mini-batch.
Args:
unused_predictions: a 2-d tensor storing the prediction scores, in which
each row represents a sample in the mini-batch and each column
represents a class.
unused_labels: a 2-d tensor storing the labels, which has the same shape
as the unused_predictions. The labels must be in the range of 0 and 1.
unused_params: loss specific parameters.
Returns:
A scalar loss tensor.
"""
raise NotImplementedError()
class CrossEntropyLoss(BaseLoss):
"""Calculate the cross entropy loss between the predictions and labels.
"""
def calculate_loss(self, predictions, labels, weights=None, **unused_params):
with tf.name_scope("loss_xent"):
epsilon = 10e-6
if FLAGS.label_smoothing:
float_labels = smoothing(labels)
else:
float_labels = tf.cast(labels, tf.float32)
cross_entropy_loss = float_labels * tf.log(predictions + epsilon) + (
1 - float_labels) * tf.log(1 - predictions + epsilon)
cross_entropy_loss = tf.negative(cross_entropy_loss)
if weights is not None:
print cross_entropy_loss, weights
weighted_loss = tf.einsum("ij,i->ij", cross_entropy_loss, weights)
print "create weighted_loss", weighted_loss
return tf.reduce_mean(tf.reduce_sum(weighted_loss, 1))
else:
return tf.reduce_mean(tf.reduce_sum(cross_entropy_loss, 1))
class HingeLoss(BaseLoss):
"""Calculate the hinge loss between the predictions and labels.
Note the subgradient is used in the backpropagation, and thus the optimization
may converge slower. The predictions trained by the hinge loss are between -1
and +1.
"""
def calculate_loss(self, predictions, labels, b=1.0, **unused_params):
with tf.name_scope("loss_hinge"):
float_labels = tf.cast(labels, tf.float32)
all_zeros = tf.zeros(tf.shape(float_labels), dtype=tf.float32)
all_ones = tf.ones(tf.shape(float_labels), dtype=tf.float32)
sign_labels = tf.subtract(tf.scalar_mul(2, float_labels), all_ones)
hinge_loss = tf.maximum(
all_zeros, tf.scalar_mul(b, all_ones) - sign_labels * predictions)
return tf.reduce_mean(tf.reduce_sum(hinge_loss, 1))
class SoftmaxLoss(BaseLoss):
"""Calculate the softmax loss between the predictions and labels.
The function calculates the loss in the following way: first we feed the
predictions to the softmax activation function and then we calculate
the minus linear dot product between the logged softmax activations and the
normalized ground truth label.
It is an extension to the one-hot label. It allows for more than one positive
labels for each sample.
"""
def calculate_loss(self, predictions, labels, **unused_params):
with tf.name_scope("loss_softmax"):
epsilon = 10e-8
float_labels = tf.cast(labels, tf.float32)
# l1 normalization (labels are no less than 0)
label_rowsum = tf.maximum(
tf.reduce_sum(float_labels, 1, keep_dims=True),
epsilon)
norm_float_labels = tf.div(float_labels, label_rowsum)
softmax_outputs = tf.nn.softmax(predictions)
softmax_loss = tf.negative(tf.reduce_sum(
tf.multiply(norm_float_labels, tf.log(softmax_outputs)), 1))
return tf.reduce_mean(softmax_loss)
class MultiTaskLoss(BaseLoss):
"""This is a vitural loss
"""
def calculate_loss(self, unused_predictions, unused_labels, **unused_params):
raise NotImplementedError()
def get_support(self, labels, support_type=None):
if support_type == None:
support_type = FLAGS.support_type
if "," in support_type:
new_labels = []
for st in support_type.split(","):
new_labels.append(tf.cast(self.get_support(labels, st), dtype=tf.float32))
support_labels = tf.concat(new_labels, axis=1)
return support_labels
elif support_type == "vertical":
num_classes = FLAGS.num_classes
num_verticals = FLAGS.num_verticals
vertical_file = FLAGS.vertical_file
vertical_mapping = np.zeros([num_classes, num_verticals], dtype=np.float32)
float_labels = tf.cast(labels, dtype=tf.float32)
with open(vertical_file) as F:
for line in F:
group = map(int, line.strip().split())
if len(group) == 2:
x, y = group
vertical_mapping[x, y] = 1
vm_init = tf.constant_initializer(vertical_mapping)
vm = tf.get_variable("vm", shape = [num_classes, num_verticals],
trainable=False, initializer=vm_init)
vertical_labels = tf.matmul(float_labels, vm)
return tf.cast(vertical_labels > 0.2, tf.float32)
elif support_type == "frequent":
num_frequents = FLAGS.num_frequents
frequent_labels = tf.slice(labels, begin=[0, 0], size=[-1, num_frequents])
frequent_labels = tf.cast(frequent_labels, dtype=tf.float32)
return frequent_labels
elif support_type == "label":
float_labels = tf.cast(labels, dtype=tf.float32)
return float_labels
else:
raise NotImplementedError()
class MultiTaskCrossEntropyLoss(MultiTaskLoss):
"""Calculate the loss between the predictions and labels.
"""
def calculate_loss(self, predictions, support_predictions, labels, **unused_params):
support_labels = self.get_support(labels)
ce_loss_fn = CrossEntropyLoss()
cross_entropy_loss = ce_loss_fn.calculate_loss(predictions, labels, **unused_params)
cross_entropy_loss2 = ce_loss_fn.calculate_loss(support_predictions, support_labels, **unused_params)
return cross_entropy_loss * (1.0 - FLAGS.support_loss_percent) + cross_entropy_loss2 * FLAGS.support_loss_percent
================================================
FILE: youtube-8m-ensemble/mean_average_precision_calculator.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Calculate the mean average precision.
It provides an interface for calculating mean average precision
for an entire list or the top-n ranked items.
Example usages:
We first call the function accumulate many times to process parts of the ranked
list. After processing all the parts, we call peek_map_at_n
to calculate the mean average precision.
```
import random
p = np.array([[random.random() for _ in xrange(50)] for _ in xrange(1000)])
a = np.array([[random.choice([0, 1]) for _ in xrange(50)]
for _ in xrange(1000)])
# mean average precision for 50 classes.
calculator = mean_average_precision_calculator.MeanAveragePrecisionCalculator(
num_class=50)
calculator.accumulate(p, a)
aps = calculator.peek_map_at_n()
```
"""
import numpy
import average_precision_calculator
class MeanAveragePrecisionCalculator(object):
"""This class is to calculate mean average precision.
"""
def __init__(self, num_class):
"""Construct a calculator to calculate the (macro) average precision.
Args:
num_class: A positive Integer specifying the number of classes.
top_n_array: A list of positive integers specifying the top n for each
class. The top n in each class will be used to calculate its average
precision at n.
The size of the array must be num_class.
Raises:
ValueError: An error occurred when num_class is not a positive integer;
or the top_n_array is not a list of positive integers.
"""
if not isinstance(num_class, int) or num_class <= 1:
raise ValueError("num_class must be a positive integer.")
self._ap_calculators = [] # member of AveragePrecisionCalculator
self._num_class = num_class # total number of classes
for i in range(num_class):
self._ap_calculators.append(
average_precision_calculator.AveragePrecisionCalculator())
def accumulate(self, predictions, actuals, num_positives=None):
"""Accumulate the predictions and their ground truth labels.
Args:
predictions: A list of lists storing the prediction scores. The outer
dimension corresponds to classes.
actuals: A list of lists storing the ground truth labels. The dimensions
should correspond to the predictions input. Any value
larger than 0 will be treated as positives, otherwise as negatives.
num_positives: If provided, it is a list of numbers representing the
number of true positives for each class. If not provided, the number of
true positives will be inferred from the 'actuals' array.
Raises:
ValueError: An error occurred when the shape of predictions and actuals
does not match.
"""
if not num_positives:
num_positives = [None for i in predictions.shape[1]]
calculators = self._ap_calculators
for i in range(len(predictions)):
calculators[i].accumulate(predictions[i], actuals[i], num_positives[i])
def clear(self):
for calculator in self._ap_calculators:
calculator.clear()
def is_empty(self):
return ([calculator.heap_size for calculator in self._ap_calculators] ==
[0 for _ in range(self._num_class)])
def peek_map_at_n(self):
"""Peek the non-interpolated mean average precision at n.
Returns:
An array of non-interpolated average precision at n (default 0) for each
class.
"""
aps = [self._ap_calculators[i].peek_ap_at_n()
for i in range(self._num_class)]
return aps
================================================
FILE: youtube-8m-ensemble/model_selection_scripts/.vimrc
================================================
set tabstop=2
set shiftwidth=2
set expandtab
set autoindent
================================================
FILE: youtube-8m-ensemble/model_selection_scripts/extend-step-mean_model.sh
================================================
#!/bin/bash
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
model_name="$1"
candidates_conf="$2"
train_path=/Youtube-8M/model_predictions_for_selection/ensemble_train
model_path="${DIR}/../../model/${model_name}"
all_models_conf="${model_path}/all_models.conf"
for candidates in $(cat $candidates_conf); do
echo "$candidates"
train_data_patterns=$(python ${DIR}/get_patterns.py --train_path="$train_path" --candidates="$candidates")
CUDA_VISIBLE_DEVICES=1 python ${DIR}/../eval.py \
--model_checkpoint_path="${model_path}/model.ckpt-0" \
--train_dir="${model_path}" \
--model="MeanModel" \
--echo_gap=True \
--batch_size=1024 \
--eval_data_patterns="$train_data_patterns" | tail -n 1
done
================================================
FILE: youtube-8m-ensemble/model_selection_scripts/get_extend_candidates.py
================================================
import sys
from tensorflow import flags
FLAGS = flags.FLAGS
if __name__=="__main__":
flags.DEFINE_string("top_k_file", "", "The file that contains the top-k ensemble models.")
flags.DEFINE_string("all_models_conf", "", "The file that contains all available single models.")
if __name__=="__main__":
all_models = [line.strip() for line in open(FLAGS.all_models_conf) if len(line.strip()) > 0]
extend_candidates = set()
with open(FLAGS.top_k_file) as F:
ensemble_models = [line.strip().split(",") for line in F.readlines() if len(line.strip()) > 0]
for em in ensemble_models:
for model in all_models:
if model not in em:
new_combination = ",".join(sorted(em + [model]))
if new_combination not in extend_candidates:
extend_candidates.add(new_combination)
for candidate in extend_candidates:
print candidate
================================================
FILE: youtube-8m-ensemble/model_selection_scripts/get_patterns.py
================================================
import sys
from tensorflow import flags
FLAGS = flags.FLAGS
if __name__=="__main__":
flags.DEFINE_string("train_path", "", "The directory where training files locates.")
flags.DEFINE_string("candidates", "", "The candidate methods.")
if __name__=="__main__":
candidate_methods = map(lambda x: x.strip(), FLAGS.candidates.strip().split(","))
train_path = FLAGS.train_path
output_path = ",".join(map(lambda x: "%s/%s/*.tfrecord"%(train_path, x), candidate_methods))
sys.stdout.write(output_path)
sys.stdout.flush()
================================================
FILE: youtube-8m-ensemble/model_selection_scripts/get_top_k.py
================================================
import sys
from tensorflow import flags
FLAGS = flags.FLAGS
if __name__=="__main__":
flags.DEFINE_string("log_file", "", "The file that log models performances.")
flags.DEFINE_string("sorted_log_file", "", "The file that log models performances (sorted by GAP).")
flags.DEFINE_integer("top_k", 10, "The number of top models reserved.")
if __name__=="__main__":
log_file = FLAGS.log_file
with open(log_file) as F:
lines = F.readlines()
models = map(lambda x: x.strip(), lines[::2])
perfs = map(lambda x: float(x.strip().split("=")[-1]), lines[1::2])
perfs = perfs[:len(models)]
model_perfs = sorted(zip(perfs, models), reverse=True)
with open(FLAGS.sorted_log_file, "w") as Fo:
Fo.writelines(["%f\t%s\n"%(x,y) for x,y in model_perfs])
for perf, model in model_perfs[:FLAGS.top_k]:
print model
================================================
FILE: youtube-8m-ensemble/model_selection_scripts/greedy-selection-mean_model.sh
================================================
#!/bin/bash
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
model_name=$1
extend_step_script="${DIR}/extend-step-mean_model.sh"
train_path=/Youtube-8M/model_predictions_for_selection/ensemble_train
model_path="${DIR}/../../model/${model_name}"
all_models_conf="${model_path}/all_models.conf"
if [ -f $all_models_conf ]; then
top_models_conf="${all_models_conf}"
len_k_models_conf="${all_models_conf}"
for step in {14..30}; do
if [ $step -gt 1 ]; then
top_models_conf="${model_path}/top_$((${step}-1))_models.conf"
len_k_models_conf="${model_path}/len_${step}_models.conf"
echo $top_models_conf $len_k_models_conf
python ${DIR}/get_extend_candidates.py --top_k_file="$top_models_conf" --all_models_conf="${all_models_conf}" > $len_k_models_conf
fi
bash $extend_step_script $model_name ${len_k_models_conf} > ${model_path}/len_${step}_models.log
python ${DIR}/get_top_k.py --top_k=2 --log_file="${model_path}/len_${step}_models.log" --sorted_log_file="${model_path}/len_${step}_models.sorted.log" > ${model_path}/top_${step}_models.conf
top_models_conf="${model_path}/top_${step}_models.conf"
done
else
echo $all_models_conf not found, did nothing
fi
================================================
FILE: youtube-8m-ensemble/model_utils.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains a collection of util functions for model construction.
"""
import numpy
import tensorflow as tf
from tensorflow import logging
from tensorflow import flags
import tensorflow.contrib.slim as slim
def SampleRandomSequence(model_input, num_frames, num_samples):
"""Samples a random sequence of frames of size num_samples.
Args:
model_input: A tensor of size batch_size x max_frames x feature_size
num_frames: A tensor of size batch_size x 1
num_samples: A scalar
Returns:
`model_input`: A tensor of size batch_size x num_samples x feature_size
"""
batch_size = tf.shape(model_input)[0]
frame_index_offset = tf.tile(
tf.expand_dims(tf.range(num_samples), 0), [batch_size, 1])
max_start_frame_index = tf.maximum(num_frames - num_samples, 0)
start_frame_index = tf.cast(
tf.multiply(
tf.random_uniform([batch_size, 1]),
tf.cast(max_start_frame_index + 1, tf.float32)), tf.int32)
frame_index = tf.minimum(start_frame_index + frame_index_offset,
tf.cast(num_frames - 1, tf.int32))
batch_index = tf.tile(
tf.expand_dims(tf.range(batch_size), 1), [1, num_samples])
index = tf.stack([batch_index, frame_index], 2)
return tf.gather_nd(model_input, index)
def SampleRandomFrames(model_input, num_frames, num_samples):
"""Samples a random set of frames of size num_samples.
Args:
model_input: A tensor of size batch_size x max_frames x feature_size
num_frames: A tensor of size batch_size x 1
num_samples: A scalar
Returns:
`model_input`: A tensor of size batch_size x num_samples x feature_size
"""
batch_size = tf.shape(model_input)[0]
frame_index = tf.cast(
tf.multiply(
tf.random_uniform([batch_size, num_samples]),
tf.tile(tf.cast(num_frames, tf.float32), [1, num_samples])), tf.int32)
batch_index = tf.tile(
tf.expand_dims(tf.range(batch_size), 1), [1, num_samples])
index = tf.stack([batch_index, frame_index], 2)
return tf.gather_nd(model_input, index)
def FramePooling(frames, method, **unused_params):
"""Pools over the frames of a video.
Args:
frames: A tensor with shape [batch_size, num_frames, feature_size].
method: "average", "max", "attention", or "none".
Returns:
A tensor with shape [batch_size, feature_size] for average, max, or
attention pooling. A tensor with shape [batch_size*num_frames, feature_size]
for none pooling.
Raises:
ValueError: if method is other than "average", "max", "attention", or
"none".
"""
if method == "average":
return tf.reduce_mean(frames, 1)
elif method == "max":
return tf.reduce_max(frames, 1)
elif method == "none":
feature_size = frames.shape_as_list()[2]
return tf.reshape(frames, [-1, feature_size])
else:
raise ValueError("Unrecognized pooling method: %s" % method)
================================================
FILE: youtube-8m-ensemble/models.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains the base class for models."""
class BaseModel(object):
"""Inherit from this class when implementing new models."""
def create_model(self, unused_model_input, **unused_params):
raise NotImplementedError()
================================================
FILE: youtube-8m-ensemble/readers.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Provides readers configured for different datasets."""
import sys
import tensorflow as tf
import utils
from tensorflow import logging
def resize_axis(tensor, axis, new_size, fill_value=0):
tensor = tf.convert_to_tensor(tensor)
shape = tf.unstack(tf.shape(tensor))
pad_shape = shape[:]
pad_shape[axis] = tf.maximum(0, new_size - shape[axis])
shape[axis] = tf.minimum(shape[axis], new_size)
shape = tf.stack(shape)
resized = tf.concat([
tf.slice(tensor, tf.zeros_like(shape), shape),
tf.fill(tf.stack(pad_shape), tf.cast(fill_value, tensor.dtype))
], axis)
# Update shape.
new_shape = tensor.get_shape().as_list() # A copy is being made.
new_shape[axis] = new_size
resized.set_shape(new_shape)
return resized
class BaseReader(object):
"""Inherit from this class when implementing new readers."""
def prepare_reader(self, unused_filename_queue):
"""Create a thread for generating prediction and label tensors."""
raise NotImplementedError()
class EnsembleReader(BaseReader):
def __init__(self,
num_classes=4716,
feature_sizes=[1024],
feature_names=["mean_inc3"]):
assert len(feature_names) == len(feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(feature_names), len(feature_sizes))
self.num_classes = num_classes
self.feature_sizes = feature_sizes
self.feature_names = feature_names
def prepare_reader(self, filename_queue, batch_size=1024):
reader = tf.TFRecordReader()
_, serialized_examples = reader.read_up_to(filename_queue, batch_size)
# set the mapping from the fields to data types in the proto
num_features = len(self.feature_names)
assert num_features > 0, "self.feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(self.feature_names), len(self.feature_sizes))
feature_map = {"video_id": tf.FixedLenFeature([], tf.string),
"labels": tf.VarLenFeature(tf.int64)}
for feature_index in range(num_features):
feature_map[self.feature_names[feature_index]] = tf.FixedLenFeature(
[self.feature_sizes[feature_index]], tf.float32)
features = tf.parse_example(serialized_examples, features=feature_map)
labels = tf.sparse_to_indicator(features["labels"], self.num_classes)
labels.set_shape([None, self.num_classes])
concatenated_features = tf.concat([
features[feature_name] for feature_name in self.feature_names], 1)
return features["video_id"], concatenated_features, labels, tf.ones([tf.shape(serialized_examples)[0]])
class EnsembleFrameReader(BaseReader):
def __init__(self,
num_classes=4716,
feature_sizes=[1024],
feature_names=["mean_inc3"],
max_frames=300):
assert len(feature_names) == len(feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(feature_names), len(feature_sizes))
self.num_classes = num_classes
self.feature_sizes = feature_sizes
self.feature_names = feature_names
self.max_frames = max_frames
def get_video_matrix(self,
features,
feature_size,
max_frames):
reshaped_features = tf.reshape(features, [-1])
num_frames = tf.minimum(tf.shape(reshaped_features)[0], max_frames)
feature_matrix = resize_axis(reshaped_features, 0, max_frames, fill_value="")
return feature_matrix, num_frames
def prepare_reader(self, filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
contexts, features = tf.parse_single_sequence_example(
serialized_example,
context_features={
"video_id": tf.FixedLenFeature([], tf.string),
"labels": tf.VarLenFeature(tf.int64)},
sequence_features={
"rgb": tf.FixedLenSequenceFeature([], dtype=tf.string),
"audio": tf.FixedLenSequenceFeature([], dtype=tf.string),
})
# read ground truth labels
labels = (tf.cast(
tf.sparse_to_dense(contexts["labels"].values, (self.num_classes,), 1,
validate_indices=False),
tf.bool))
rgbs, num_frames = self.get_video_matrix(features["rgb"], 1024, self.max_frames)
audios, num_frames = self.get_video_matrix(features["audio"], 1024, self.max_frames)
batch_video_ids = tf.expand_dims(contexts["video_id"], 0)
batch_rgbs = tf.expand_dims(rgbs, 0)
batch_audios = tf.expand_dims(audios, 0)
batch_labels = tf.expand_dims(labels, 0)
batch_frames = tf.expand_dims(num_frames, 0)
return batch_video_ids, batch_rgbs, batch_audios, batch_labels, batch_frames
================================================
FILE: youtube-8m-ensemble/top_k_scripts/eval-attention_matrix_model.sh
================================================
model=$1
conf=$2
moe=$3
att=$4
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
validate_path=/Youtube-8M/model_predictions/ensemble_validate
validate_data_patterns=""
for d in $(cat $conf); do
validate_data_patterns="${validate_path}/${d}/*.tfrecord${validate_data_patterns:+,$validate_data_patterns}"
done
echo "$validate_data_patterns"
input_data_pattern="${validate_path}/model_input/*.tfrecord"
start=0
EVERY=300
DIR="$(pwd)"
MODEL_DIR="${DIR}/../model/${model}" \
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES="$GPU_ID" python eval.py \
--model_checkpoint_path="../model/${model}/model.ckpt-${checkpoint}" \
--train_dir="../model/${model}" \
--model="AttentionMatrixModel" \
--moe_num_mixtures=$moe \
--attention_matrix_rank=$att \
--eval_data_patterns="$validate_data_patterns" \
--input_data_pattern="$input_data_pattern"
fi
done
================================================
FILE: youtube-8m-ensemble/top_k_scripts/infer-attention_matrix_model.sh
================================================
model=$1
conf=$2
checkpoint=$3
test_path=/Youtube-8M/model_predictions/test
test_data_patterns=""
for d in $(cat $conf); do
test_data_patterns="${test_path}/${d}/*.tfrecord${test_data_patterns:+,$test_data_patterns}"
done
echo "$test_data_patterns"
input_data_pattern="${test_path}/model_input/*.tfrecord"
#CUDA_VISIBLE_DEVICES=0 python inference.py \
python inference.py \
--model_checkpoint_path="../model/${model}/model.ckpt-${checkpoint}" \
--output_file="../model/${model}/predictions.${model}.csv" \
--model="AttentionMatrixModel" \
--moe_num_mixtures=4 \
--attention_matrix_rank=8 \
--batch_size=1024 \
--input_data_pattern="$input_data_pattern" \
--input_data_patterns="$test_data_patterns"
================================================
FILE: youtube-8m-ensemble/top_k_scripts/preensemble-attention_matrix_model.sh
================================================
model=$1
conf=$2
part=$3
checkpoint=$4
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
test_path=/Youtube-8M/model_predictions/${part}
test_data_patterns=""
for d in $(cat $conf); do
test_data_patterns="${test_path}/${d}/*.tfrecord${test_data_patterns:+,$test_data_patterns}"
done
echo "$test_data_patterns"
input_data_pattern="${test_path}/model_input/*.tfrecord"
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/${model}" \
--model_checkpoint_path="../model/${model}/model.ckpt-${checkpoint}" \
--input_data_patterns="$test_data_patterns" \
--input_data_pattern="$input_data_pattern" \
--model="AttentionMatrixModel" \
--moe_num_mixtures=4 \
--attention_matrix_rank=8 \
--batch_size=1024 \
--file_size=4096
================================================
FILE: youtube-8m-ensemble/top_k_scripts/run_top_k.sh
================================================
#!/bin/bash
for i in 8 12 16 20; do
bash top_k_scripts/train-attention_matrix_model.sh model_selection/top_${i}_model ../model/model_selection/top_${i}_model.conf 4 $(($i/4))
bash top_k_scripts/eval-attention_matrix_model.sh model_selection/top_${i}_model ../model/model_selection/top_${i}_model.conf 4 $(($i/4))
done
================================================
FILE: youtube-8m-ensemble/top_k_scripts/train-attention_matrix_model.sh
================================================
model=$1
conf=$2
moe=$3
att=$4
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
train_path=/Youtube-8M/model_predictions/ensemble_train
train_data_patterns=""
for d in $(cat $conf); do
train_data_patterns="${train_path}/${d}/*.tfrecord${train_data_patterns:+,$train_data_patterns}"
done
echo $train_data_patterns
input_data_pattern="${train_path}/model_input/*.tfrecord"
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="../model/${model}" \
--train_data_patterns="$train_data_patterns" \
--input_data_pattern="$input_data_pattern" \
--model=AttentionMatrixModel \
--base_learning_rate=0.004 \
--moe_num_mixtures=${moe} \
--attention_matrix_rank=${att} \
--keep_checkpoint_every_n_hours=0.1 \
--batch_size=1024 \
--num_epochs=4
================================================
FILE: youtube-8m-ensemble/train.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for training Tensorflow models on the YouTube-8M dataset."""
import json
import os
import time
import numpy
import numpy as np
import eval_util
import losses
import ensemble_level_models
import readers
import tensorflow as tf
import tensorflow.contrib.slim as slim
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import utils
FLAGS = flags.FLAGS
if __name__ == "__main__":
# Dataset flags.
flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
"The directory to save the model files in.")
flags.DEFINE_string(
"train_data_patterns", "",
"Multiple file globs for the training dataset.")
flags.DEFINE_string(
"input_data_pattern", None,
"File globs for original model input.")
flags.DEFINE_string("feature_names", "predictions", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "4716", "Length of the feature vectors.")
# Model flags.
flags.DEFINE_string(
"model", "LogisticModel",
"Which architecture to use for the model. Models are defined "
"in models.py.")
flags.DEFINE_bool(
"multitask", False,
"Whether to consider support_predictions")
flags.DEFINE_bool(
"start_new_model", False,
"If set, this will not resume from a checkpoint and will instead create a"
" new model instance.")
# Training flags.
flags.DEFINE_integer("batch_size", 256,
"How many examples to process per batch for training.")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Which loss function to use for training the model.")
flags.DEFINE_float(
"regularization_penalty", 1,
"How much weight to give to the regularization loss (the label loss has "
"a weight of 1).")
flags.DEFINE_float("base_learning_rate", 0.01,
"Which learning rate to start with.")
flags.DEFINE_float("learning_rate_decay", 0.95,
"Learning rate decay factor to be applied every "
"learning_rate_decay_examples.")
flags.DEFINE_float("learning_rate_decay_examples", 1000000,
"Multiply current learning rate by learning_rate_decay "
"every learning_rate_decay_examples.")
flags.DEFINE_integer("num_epochs", 10,
"How many passes to make over the dataset before "
"halting training.")
flags.DEFINE_float("keep_checkpoint_every_n_hours", 0.1,
"How many hours before saving a new checkpoint")
flags.DEFINE_bool("reweight", False,
"Whether to load model weight from file.")
flags.DEFINE_string("sample_vocab_file", "",
"Where to load video_id vocabulary.")
flags.DEFINE_string("sample_freq_file", "",
"Where to load sample frequency.")
# Other flags.
flags.DEFINE_string("optimizer", "AdamOptimizer",
"What optimizer class to use.")
flags.DEFINE_float("clip_gradient_norm", 1.0, "Norm to clip gradients to.")
flags.DEFINE_bool(
"log_device_placement", False,
"Whether to write the device on which every op will run into the "
"logs on startup.")
flags.DEFINE_integer("recall_at_n", 100,
"N in recall@N.")
flags.DEFINE_bool("training", True,
"Whether to train")
flags.DEFINE_bool(
"dropout", False,
"Whether to consider dropout")
flags.DEFINE_float("keep_prob", 1.0,
"probability to keep output (used in dropout, keep it unchanged in validationg and test)")
flags.DEFINE_float("noise_level", 0.0,
"standard deviation of noise (added to hidden nodes)")
def get_input_data_tensors(reader,
data_pattern,
batch_size=256,
num_epochs=None):
logging.info("Using batch size of " + str(batch_size) + " for training.")
with tf.name_scope("train_input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find training files. data_pattern='" +
data_pattern + "'.")
logging.info("Number of training files: %s.", str(len(files)))
files.sort()
filename_queue = tf.train.string_input_producer(
files, num_epochs=num_epochs, shuffle=False)
training_data = reader.prepare_reader(filename_queue)
return tf.train.batch(
training_data,
batch_size=batch_size,
capacity=FLAGS.batch_size * 4,
allow_smaller_final_batch=True,
enqueue_many=True)
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def get_video_weights_array():
weight_lines = open(FLAGS.sample_freq_file).readlines()
weights = numpy.array(map(float, weight_lines))
weights = weights.reshape([len(weight_lines)])
return weights, len(weight_lines)
def optional_assign_weights(sess, weights_input, weights_assignment):
if weights_input is not None:
weights, length = get_video_weights_array()
_ = sess.run(weights_assignment, feed_dict={weights_input: weights})
print "Assigned weights from %s" % FLAGS.sample_freq_file
else:
print "Collection weights_input not found"
def get_video_weights(video_id_batch):
video_id_to_index = tf.contrib.lookup.string_to_index_table_from_file(
vocabulary_file=FLAGS.sample_vocab_file, default_value=0)
indexes = video_id_to_index.lookup(video_id_batch)
weights, length = get_video_weights_array()
weights_input = tf.placeholder(tf.float32, shape=[length], name="sample_weights_input")
weights_tensor = tf.get_variable("sample_weights",
shape=[length],
trainable=False,
dtype=tf.float32,
initializer=tf.constant_initializer(weights))
weights_assignment = tf.assign(weights_tensor, weights_input)
tf.add_to_collection("weights_input", weights_input)
tf.add_to_collection("weights_assignment", weights_assignment)
video_weight_batch = tf.nn.embedding_lookup(weights_tensor, indexes)
return video_weight_batch
def build_graph(all_readers,
all_train_data_patterns,
input_reader,
input_data_pattern,
model,
label_loss_fn=losses.CrossEntropyLoss(),
batch_size=256,
base_learning_rate=0.01,
learning_rate_decay_examples=1000000,
learning_rate_decay=0.95,
optimizer_class=tf.train.AdamOptimizer,
clip_gradient_norm=1.0,
regularization_penalty=1,
num_epochs=None):
"""Creates the Tensorflow graph.
This will only be called once in the life of
a training model, because after the graph is created the model will be
restored from a meta graph file rather than being recreated.
Args:
all_readers: The data file readers. Every element in it should inherit from BaseReader.
model: The core model (e.g. logistic or neural net). It should inherit
from BaseModel.
train_data_patterns: glob paths to the training data files.
label_loss_fn: What kind of loss to apply to the model. It should inherit
from BaseLoss.
batch_size: How many examples to process at a time.
base_learning_rate: What learning rate to initialize the optimizer with.
optimizer_class: Which optimization algorithm to use.
clip_gradient_norm: Magnitude of the gradient to clip to.
regularization_penalty: How much weight to give the regularization loss
compared to the label loss.
num_epochs: How many passes to make over the data. 'None' means an
unlimited number of passes.
"""
global_step = tf.Variable(0, trainable=False, name="global_step")
learning_rate = tf.train.exponential_decay(
base_learning_rate,
global_step * batch_size,
learning_rate_decay_examples,
learning_rate_decay,
staircase=True)
tf.summary.scalar('learning_rate', learning_rate)
original_input = None
if input_data_pattern is not None:
original_video_id, original_input, unused_labels_batch, unused_num_frames = (
get_input_data_tensors(
input_reader,
input_data_pattern,
batch_size=batch_size,
num_epochs=num_epochs))
optimizer = optimizer_class(learning_rate)
model_input_raw_tensors = []
labels_batch_tensor = None
for reader, data_pattern in zip(all_readers, all_train_data_patterns):
video_id, model_input_raw, labels_batch, unused_num_frames = (
get_input_data_tensors(
reader,
data_pattern,
batch_size=batch_size,
num_epochs=num_epochs))
if labels_batch_tensor is None:
labels_batch_tensor = labels_batch
model_input_raw_tensors.append(tf.expand_dims(model_input_raw, axis=2))
if original_input is not None:
id_match = tf.ones_like(original_video_id, dtype=tf.float32)
id_match = id_match * tf.cast(tf.equal(original_video_id, video_id), dtype=tf.float32)
tf.summary.scalar("model/id_match", tf.reduce_mean(id_match))
model_input = tf.concat(model_input_raw_tensors, axis=2)
labels_batch = labels_batch_tensor
tf.summary.histogram("model/input", model_input)
with tf.name_scope("model"):
if FLAGS.noise_level > 0:
noise_level_tensor = tf.placeholder_with_default(0.0, shape=[], name="noise_level")
else:
noise_level_tensor = None
if FLAGS.dropout:
keep_prob_tensor = tf.placeholder_with_default(1.0, shape=[], name="keep_prob")
result = model.create_model(
model_input,
labels=labels_batch,
vocab_size=reader.num_classes,
original_input=original_input,
dropout=FLAGS.dropout,
keep_prob=keep_prob_tensor,
noise_level=noise_level_tensor)
else:
result = model.create_model(
model_input,
labels=labels_batch,
vocab_size=reader.num_classes,
original_input=original_input,
noise_level=noise_level_tensor)
for variable in slim.get_model_variables():
tf.summary.histogram(variable.op.name, variable)
predictions = result["predictions"]
if "loss" in result.keys():
label_loss = result["loss"]
else:
video_weights_batch = None
if FLAGS.reweight:
video_weights_batch = get_video_weights(video_id)
else:
video_weights_batch = None
if FLAGS.multitask:
print "using multitask loss"
support_predictions = result["support_predictions"]
tf.summary.histogram("model/support_predictions", support_predictions)
print "support_predictions", support_predictions
label_loss = label_loss_fn.calculate_loss(predictions, support_predictions, labels_batch, weights=video_weights_batch)
else:
print "using original loss"
label_loss = label_loss_fn.calculate_loss(predictions, labels_batch, weights=video_weights_batch)
tf.summary.histogram("model/predictions", predictions)
tf.summary.scalar("label_loss", label_loss)
if "regularization_loss" in result.keys():
reg_loss = result["regularization_loss"]
else:
reg_loss = tf.constant(0.0)
reg_losses = tf.losses.get_regularization_losses()
if reg_losses:
reg_loss += tf.add_n(reg_losses)
if regularization_penalty != 0:
tf.summary.scalar("reg_loss", reg_loss)
# Adds update_ops (e.g., moving average updates in batch normalization) as
# a dependency to the train_op.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
if "update_ops" in result.keys():
update_ops += result["update_ops"]
if update_ops:
with tf.control_dependencies(update_ops):
barrier = tf.no_op(name="gradient_barrier")
with tf.control_dependencies([barrier]):
label_loss = tf.identity(label_loss)
# Incorporate the L2 weight penalties etc.
final_loss = regularization_penalty * reg_loss + label_loss
if FLAGS.training:
gradients = optimizer.compute_gradients(final_loss,
colocate_gradients_with_ops=False)
if clip_gradient_norm > 0:
with tf.name_scope('clip_grads'):
gradients = utils.clip_gradient_norms(gradients , clip_gradient_norm)
train_op = optimizer.apply_gradients(gradients, global_step=global_step)
else:
train_op = tf.no_op()
tf.add_to_collection("global_step", global_step)
tf.add_to_collection("loss", label_loss)
tf.add_to_collection("predictions", predictions)
tf.add_to_collection("input_batch_raw", model_input)
tf.add_to_collection("input_batch", model_input)
tf.add_to_collection("labels", tf.cast(labels_batch, tf.float32))
tf.add_to_collection("train_op", train_op)
if FLAGS.dropout:
tf.add_to_collection("keep_prob", keep_prob_tensor)
if FLAGS.noise_level > 0:
tf.add_to_collection("noise_level", noise_level_tensor)
class Trainer(object):
"""A Trainer to train a Tensorflow graph."""
def __init__(self, cluster, task, train_dir, log_device_placement=True):
""""Creates a Trainer.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
self.cluster = cluster
self.task = task
self.is_master = (task.type == "master" and task.index == 0)
self.train_dir = train_dir
self.config = tf.ConfigProto(log_device_placement=log_device_placement)
if self.is_master and self.task.index > 0:
raise StandardError("%s: Only one replica of master expected",
task_as_string(self.task))
def run(self, start_new_model=False):
"""Performs training on the currently defined Tensorflow graph.
Returns:
A tuple of the training Hit@1 and the training PERR.
"""
if self.is_master and start_new_model:
self.remove_training_directory(self.train_dir)
target, device_fn = self.start_server_if_distributed()
meta_filename = self.get_meta_filename(start_new_model, self.train_dir)
with tf.Graph().as_default() as graph:
if meta_filename:
saver = self.recover_model(meta_filename)
with tf.device(device_fn):
if not meta_filename:
saver = self.build_model()
global_step = tf.get_collection("global_step")[0]
loss = tf.get_collection("loss")[0]
predictions = tf.get_collection("predictions")[0]
labels = tf.get_collection("labels")[0]
train_op = tf.get_collection("train_op")[0]
init_op = tf.global_variables_initializer()
if FLAGS.dropout:
keep_prob_tensor = tf.get_collection("keep_prob")[0]
if FLAGS.noise_level > 0:
noise_level_tensor = tf.get_collection("noise_level")[0]
if FLAGS.reweight:
weights_input, weights_assignment = None, None
if len(tf.get_collection("weights_input")) > 0:
weights_input = tf.get_collection("weights_input")[0]
weights_assignment = tf.get_collection("weights_assignment")[0]
sv = tf.train.Supervisor(
graph,
logdir=self.train_dir,
init_op=init_op,
is_chief=self.is_master,
global_step=global_step,
save_model_secs=6 * 60,
save_summaries_secs=120,
saver=saver)
logging.info("%s: Starting managed session.", task_as_string(self.task))
with sv.managed_session(target, config=self.config) as sess:
# re-assign weights
if FLAGS.reweight:
optional_assign_weights(sess, weights_input, weights_assignment)
try:
logging.info("%s: Entering training loop.", task_as_string(self.task))
while not sv.should_stop():
batch_start_time = time.time()
custom_feed = {}
if FLAGS.dropout:
custom_feed[keep_prob_tensor] = FLAGS.keep_prob
if FLAGS.noise_level > 0:
custom_feed[noise_level_tensor] = FLAGS.noise_level
_, global_step_val, loss_val, predictions_val, labels_val = sess.run(
[train_op, global_step, loss, predictions, labels], feed_dict=custom_feed)
seconds_per_batch = time.time() - batch_start_time
if self.is_master:
examples_per_second = labels_val.shape[0] / seconds_per_batch
hit_at_one = eval_util.calculate_hit_at_one(predictions_val,
labels_val)
perr = eval_util.calculate_precision_at_equal_recall_rate(
predictions_val, labels_val)
recall = "N/A"
if False:
recall = eval_util.calculate_recall_at_n(
predictions_val, labels_val, FLAGS.recall_at_n)
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_Recall@%d" % FLAGS.recall_at_n, recall), global_step_val)
recall = "%.2f" % recall
gap = eval_util.calculate_gap(predictions_val, labels_val)
logging.info(
"%s: training step " + str(global_step_val) + "| Hit@1: " +
("%.2f" % hit_at_one) + " PERR: " + ("%.2f" % perr) + " GAP: " +
("%.2f" % gap) + " Recall@%d: " % FLAGS.recall_at_n +
recall + " Loss: " + str(loss_val),
task_as_string(self.task))
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_Hit@1", hit_at_one),
global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_Perr", perr), global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_GAP", gap), global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("global_step/Examples/Second",
examples_per_second), global_step_val)
sv.summary_writer.flush()
except tf.errors.OutOfRangeError:
logging.info("%s: Done training -- epoch limit reached.",
task_as_string(self.task))
logging.info("%s: Exited training loop.", task_as_string(self.task))
sv.Stop()
def start_server_if_distributed(self):
"""Starts a server if the execution is distributed."""
if self.cluster:
logging.info("%s: Starting trainer within cluster %s.",
task_as_string(self.task), self.cluster.as_dict())
server = start_server(self.cluster, self.task)
target = server.target
device_fn = tf.train.replica_device_setter(
ps_device="/job:ps",
worker_device="/job:%s/task:%d" % (self.task.type, self.task.index),
cluster=self.cluster)
else:
target = ""
device_fn = ""
return (target, device_fn)
def remove_training_directory(self, train_dir):
"""Removes the training directory."""
try:
logging.info(
"%s: Removing existing train directory.",
task_as_string(self.task))
gfile.DeleteRecursively(train_dir)
except:
logging.error(
"%s: Failed to delete directory " + train_dir +
" when starting a new model. Please delete it manually and" +
" try again.", task_as_string(self.task))
def get_meta_filename(self, start_new_model, train_dir):
if start_new_model:
logging.info("%s: Flag 'start_new_model' is set. Building a new model.",
task_as_string(self.task))
return None
latest_checkpoint = tf.train.latest_checkpoint(train_dir)
if not latest_checkpoint:
logging.info("%s: No checkpoint file found. Building a new model.",
task_as_string(self.task))
return None
meta_filename = latest_checkpoint + ".meta"
if not gfile.Exists(meta_filename):
logging.info("%s: No meta graph file found. Building a new model.",
task_as_string(self.task))
return None
else:
return meta_filename
def recover_model(self, meta_filename):
logging.info("%s: Restoring from meta graph file %s",
task_as_string(self.task), meta_filename)
return tf.train.import_meta_graph(meta_filename)
def build_model(self):
"""Find the model and build the graph."""
# Convert feature_names and feature_sizes to lists of values.
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
# prepare a reader for each single model prediction result
all_readers = []
all_patterns = FLAGS.train_data_patterns
all_patterns = map(lambda x: x.strip(), all_patterns.strip().strip(",").split(","))
for i in xrange(len(all_patterns)):
all_readers.append(readers.EnsembleReader(
feature_names=feature_names, feature_sizes=feature_sizes))
input_reader = None
input_data_pattern = None
if FLAGS.input_data_pattern is not None:
input_reader = readers.EnsembleReader(
feature_names=["input"], feature_sizes=[1024+128])
input_data_pattern = FLAGS.input_data_pattern
# Find the model.
model = find_class_by_name(FLAGS.model, [ensemble_level_models])()
label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])()
optimizer_class = find_class_by_name(FLAGS.optimizer, [tf.train])
build_graph(all_readers=all_readers,
all_train_data_patterns=all_patterns,
input_reader=input_reader,
input_data_pattern=input_data_pattern,
model=model,
optimizer_class=optimizer_class,
clip_gradient_norm=FLAGS.clip_gradient_norm,
label_loss_fn=label_loss_fn,
base_learning_rate=FLAGS.base_learning_rate,
learning_rate_decay=FLAGS.learning_rate_decay,
learning_rate_decay_examples=FLAGS.learning_rate_decay_examples,
regularization_penalty=FLAGS.regularization_penalty,
batch_size=FLAGS.batch_size,
num_epochs=FLAGS.num_epochs)
logging.info("%s: Built graph.", task_as_string(self.task))
return tf.train.Saver(max_to_keep=3, keep_checkpoint_every_n_hours=FLAGS.keep_checkpoint_every_n_hours)
class ParameterServer(object):
"""A parameter server to serve variables in a distributed execution."""
def __init__(self, cluster, task):
"""Creates a ParameterServer.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
self.cluster = cluster
self.task = task
def run(self):
"""Starts the parameter server."""
logging.info("%s: Starting parameter server within cluster %s.",
task_as_string(self.task), self.cluster.as_dict())
server = start_server(self.cluster, self.task)
server.join()
def start_server(cluster, task):
"""Creates a Server.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
if not task.type:
raise ValueError("%s: The task type must be specified." %
task_as_string(task))
if task.index is None:
raise ValueError("%s: The task index must be specified." %
task_as_string(task))
# Create and start a server.
return tf.train.Server(
tf.train.ClusterSpec(cluster),
protocol="grpc",
job_name=task.type,
task_index=task.index)
def task_as_string(task):
return "/job:%s/task:%s" % (task.type, task.index)
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object,), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.",
task_as_string(task), tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
Trainer(cluster, task, FLAGS.train_dir, FLAGS.log_device_placement).run(
start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-ensemble/training_utils/del.py
================================================
import os
check = {}
check_list = []
for filename in os.listdir("."):
if filename.endswith("meta"):
checkpoint = int(filename.split("-")[1].split(".")[0])
check_list.append(checkpoint)
check_list.sort()
for checkpoint in check_list:
if check.has_key(checkpoint / 4000):
print checkpoint
else:
check[checkpoint / 4000] = True
================================================
FILE: youtube-8m-ensemble/training_utils/sample_conf.py
================================================
import random
from datetime import datetime
import tensorflow as tf
from tensorflow import flags
FLAGS = flags.FLAGS
if __name__=="__main__":
flags.DEFINE_string("main_conf_file", "", "The conf file to sample from.")
flags.DEFINE_string("sub_conf_file", "", "The conf file to randomly generate.")
if __name__=="__main__":
with open(FLAGS.main_conf_file) as F:
methods = []
sample_methods = []
# methods
for line in F:
m = line.strip()
if m:
methods.append(m)
# random sample
random.seed(datetime.now())
for i in xrange(len(methods)):
index = random.randint(0,len(methods)-1)
m = methods[index]
sample_methods.append(m)
# output weight
with open(FLAGS.sub_conf_file, "w") as out:
out.writelines([m+"\n" for m in sample_methods])
================================================
FILE: youtube-8m-ensemble/training_utils/sample_freq.py
================================================
import random
from datetime import datetime
import tensorflow as tf
from tensorflow import flags
FLAGS = flags.FLAGS
if __name__=="__main__":
flags.DEFINE_string("video_id_file", "", "The file in which every line is a video_id.")
flags.DEFINE_string("output_freq_file", "", "Output the corresponding freq of video_ids.")
if __name__=="__main__":
with open(FLAGS.video_id_file) as F:
word_list = []
freq_dict = {}
# OOV
line = F.next()
for line in F:
word = line.strip()
if word:
word_list.append(word)
freq_dict[word] = 0
# random sample
random.seed(datetime.now())
for i in xrange(len(word_list)):
index = random.randint(0,len(word_list)-1)
word = word_list[index]
freq_dict[word] += 1
# get weight
word_weights = []
word_weights.append(1)
for word in word_list:
word_weights.append(freq_dict[word])
# output weight
with open(FLAGS.output_freq_file, "w") as out:
out.writelines([str(item)+"\n" for item in word_weights])
================================================
FILE: youtube-8m-ensemble/training_utils/select.py
================================================
import os
import sys
every = int(sys.argv[1])
check = {}
check_list = []
for filename in os.listdir("."):
if filename.endswith("meta"):
checkpoint = int(filename.split("-")[1].split(".")[0])
check_list.append(checkpoint)
check_list.sort()
for checkpoint in check_list:
if check.has_key(checkpoint / every):
pass
else:
check[checkpoint / every] = True
print checkpoint
================================================
FILE: youtube-8m-ensemble/utils.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains a collection of util functions for training and evaluating.
"""
import numpy
import tensorflow as tf
from tensorflow import logging
def Dequantize(feat_vector, max_quantized_value=2, min_quantized_value=-2):
"""Dequantize the feature from the byte format to the float format.
Args:
feat_vector: the input 1-d vector.
max_quantized_value: the maximum of the quantized value.
min_quantized_value: the minimum of the quantized value.
Returns:
A float vector which has the same shape as feat_vector.
"""
assert max_quantized_value > min_quantized_value
quantized_range = max_quantized_value - min_quantized_value
scalar = quantized_range / 255.0
bias = (quantized_range / 512.0) + min_quantized_value
return feat_vector * scalar + bias
def MakeSummary(name, value):
"""Creates a tf.Summary proto with the given name and value."""
summary = tf.Summary()
val = summary.value.add()
val.tag = str(name)
val.simple_value = float(value)
return summary
def AddGlobalStepSummary(summary_writer,
global_step_val,
global_step_info_dict,
summary_scope="Eval"):
"""Add the global_step summary to the Tensorboard.
Args:
summary_writer: Tensorflow summary_writer.
global_step_val: a int value of the global step.
global_step_info_dict: a dictionary of the evaluation metrics calculated for
a mini-batch.
summary_scope: Train or Eval.
Returns:
A string of this global_step summary
"""
this_hit_at_one = global_step_info_dict["hit_at_one"]
this_perr = global_step_info_dict["perr"]
this_loss = global_step_info_dict["loss"]
examples_per_second = global_step_info_dict.get("examples_per_second", -1)
summary_writer.add_summary(
MakeSummary("GlobalStep/" + summary_scope + "_Hit@1", this_hit_at_one),
global_step_val)
summary_writer.add_summary(
MakeSummary("GlobalStep/" + summary_scope + "_Perr", this_perr),
global_step_val)
summary_writer.add_summary(
MakeSummary("GlobalStep/" + summary_scope + "_Loss", this_loss),
global_step_val)
if examples_per_second != -1:
summary_writer.add_summary(
MakeSummary("GlobalStep/" + summary_scope + "_Example_Second",
examples_per_second), global_step_val)
summary_writer.flush()
info = ("global_step {0} | Batch Hit@1: {1:.3f} | Batch PERR: {2:.3f} | Batch Loss: {3:.3f} "
"| Examples_per_sec: {4:.3f}").format(
global_step_val, this_hit_at_one, this_perr, this_loss,
examples_per_second)
return info
def AddEpochSummary(summary_writer,
global_step_val,
epoch_info_dict,
summary_scope="Eval"):
"""Add the epoch summary to the Tensorboard.
Args:
summary_writer: Tensorflow summary_writer.
global_step_val: a int value of the global step.
epoch_info_dict: a dictionary of the evaluation metrics calculated for the
whole epoch.
summary_scope: Train or Eval.
Returns:
A string of this global_step summary
"""
epoch_id = epoch_info_dict["epoch_id"]
avg_hit_at_one = epoch_info_dict["avg_hit_at_one"]
avg_perr = epoch_info_dict["avg_perr"]
avg_loss = epoch_info_dict["avg_loss"]
aps = epoch_info_dict["aps"]
gap = epoch_info_dict["gap"]
mean_ap = numpy.mean(aps)
summary_writer.add_summary(
MakeSummary("Epoch/" + summary_scope + "_Avg_Hit@1", avg_hit_at_one),
global_step_val)
summary_writer.add_summary(
MakeSummary("Epoch/" + summary_scope + "_Avg_Perr", avg_perr),
global_step_val)
summary_writer.add_summary(
MakeSummary("Epoch/" + summary_scope + "_Avg_Loss", avg_loss),
global_step_val)
summary_writer.add_summary(
MakeSummary("Epoch/" + summary_scope + "_MAP", mean_ap),
global_step_val)
summary_writer.add_summary(
MakeSummary("Epoch/" + summary_scope + "_GAP", gap),
global_step_val)
summary_writer.flush()
info = ("epoch/eval number {0} | Avg_Hit@1: {1:.3f} | Avg_PERR: {2:.3f} "
"| MAP: {3:.3f} | GAP: {4:.3f} | Avg_Loss: {5:3f}").format(
epoch_id, avg_hit_at_one, avg_perr, mean_ap, gap, avg_loss)
return info
def GetListOfFeatureNamesAndSizes(feature_names, feature_sizes):
"""Extract the list of feature names and the dimensionality of each feature
from string of comma separated values.
Args:
feature_names: string containing comma separated list of feature names
feature_sizes: string containing comma separated list of feature sizes
Returns:
List of the feature names and list of the dimensionality of each feature.
Elements in the first/second list are strings/integers.
"""
list_of_feature_names = [
feature_names.strip() for feature_names in feature_names.split(',')]
list_of_feature_sizes = [
int(feature_sizes) for feature_sizes in feature_sizes.split(',')]
if len(list_of_feature_names) != len(list_of_feature_sizes):
logging.error("length of the feature names (=" +
str(len(list_of_feature_names)) + ") != length of feature "
"sizes (=" + str(len(list_of_feature_sizes)) + ")")
return list_of_feature_names, list_of_feature_sizes
def clip_gradient_norms(gradients_to_variables, max_norm):
clipped_grads_and_vars = []
for grad, var in gradients_to_variables:
if grad is not None:
if isinstance(grad, tf.IndexedSlices):
tmp = tf.clip_by_norm(grad.values, max_norm)
grad = tf.IndexedSlices(tmp, grad.indices, grad.dense_shape)
else:
grad = tf.clip_by_norm(grad, max_norm)
clipped_grads_and_vars.append((grad, var))
return clipped_grads_and_vars
================================================
FILE: youtube-8m-wangheda/.vimrc
================================================
set tabstop=2
set shiftwidth=2
set expandtab
set autoindent
color desert
================================================
FILE: youtube-8m-wangheda/CONTRIBUTING.md
================================================
# How to contribute
We are accepting patches and contributions to this project. To set expectations,
this project is primarily intended to be a flexible starting point for
researchers working with the YouTube-8M dataset. As such, we would like to keep
it simple. We are more likely to accept small bug fixes and optimizations, and
less likely to accept patches which add significant complexity. For the latter
type of contribution, we recommend creating a Github fork of the project
instead.
If you would like to contribute, there are a few small guidelines you need to
follow.
## Contributor License Agreement
Contributions to any Google project must be accompanied by a Contributor License
Agreement. This is necessary because you own the copyright to your changes, even
after your contribution becomes part of this project. So this agreement simply
gives us permission to use and redistribute your contributions as part of the
project. Head over to to see your current
agreements on file or to sign a new one.
You generally only need to submit a CLA once, so if you've already submitted one
(even if it was for a different project), you probably don't need to do it
again.
## Code reviews
All submissions, including submissions by project members, require review. We
use GitHub pull requests for this purpose. Consult [GitHub Help] for more
information on using pull requests.
[GitHub Help]: https://help.github.com/articles/about-pull-requests/
================================================
FILE: youtube-8m-wangheda/LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: youtube-8m-wangheda/README.md
================================================
# YouTube-8M Tensorflow Starter Code
This repo contains starter code for training and evaluating machine learning
models over the [YouTube-8M](https://research.google.com/youtube8m/) dataset.
The code gives an end-to-end working example for reading the dataset, training a
TensorFlow model, and evaluating the performance of the model. Out of the box,
you can train several [model architectures](#overview-of-models) over either
frame-level or video-level features. The code can easily be extended to train
your own custom-defined models.
It is possible to train and evaluate on YouTube-8M in two ways: on Google Cloud
or on your own machine. This README provides instructions for both.
## Table of Contents
* [Running on Google's Cloud Machine Learning Platform](#running-on-googles-cloud-machine-learning-platform)
* [Requirements](#requirements)
* [Testing Locally](#testing-locally)
* [Training on the Cloud over Video-Level Features](#training-on-video-level-features)
* [Evaluation and Inference](#evaluation-and-inference)
* [Accessing Files on Google Cloud](#accessing-files-on-google-cloud)
* [Using Frame-Level Features](#using-frame-level-features)
* [Using Audio Features](#using-audio-features)
* [Running on Your Own Machine](#running-on-your-own-machine)
* [Requirements](#requirements-1)
* [Training on Video-Level Features](#training-on-video-level-features-1)
* [Evaluation and Inference](#evaluation-and-inference-1)
* [Using Frame-Level Features](#using-frame-level-features-1)
* [Using Audio Features](#using-audio-features-1)
* [Ground-Truth Label Files](#ground-truth-label-files)
* [Overview of Models](#overview-of-models)
* [Video-Level Models](#video-level-models)
* [Frame-Level Models](#frame-level-models)
* [Overview of Files](#overview-of-files)
* [Training](#training)
* [Evaluation](#evaluation)
* [Inference](#inference)
* [Misc](#misc)
* [About This Project](#about-this-project)
## Running on Google's Cloud Machine Learning Platform
### Requirements
This option requires you to have an appropriately configured Google Cloud
Platform account. To create and configure your account, please make sure you
follow the instructions [here](https://cloud.google.com/ml/docs/how-tos/getting-set-up).
If you are participating in the Google Cloud & YouTube-8M Video Understanding
Challenge hosted on [kaggle](https://www.kaggle.com/c/youtube8m), see [these instructions](https://www.kaggle.com/c/youtube8m#getting-started-with-google-cloud) instead.
Please also verify that you have Python 2.7+ and Tensorflow 1.0.0 or higher
installed by running the following commands:
```sh
python --version
python -c 'import tensorflow as tf; print(tf.__version__)'
```
### Testing Locally
All gcloud commands should be done from the directory *immediately above* the
source code. You should be able to see the source code directory if you
run 'ls'.
As you are developing your own models, you will want to test them
quickly to flush out simple problems without having to submit them to the cloud.
You can use the `gcloud beta ml local` set of commands for that.
Here is an example command line for video-level training:
```sh
gcloud beta ml local train \
--package-path=youtube-8m --module-name=youtube-8m.train -- \
--train_data_pattern='gs://youtube8m-ml/1/video_level/train/train*.tfrecord' \
--train_dir=/tmp/yt8m_train --start_new_model
```
You might want to download some training shards locally to speed things up and
allow you to work offline. The command below will copy 10 out of the 4096
training data files to the current directory.
```sh
# Downloads 55MB of data.
gsutil cp gs://us.data.yt8m.org/1/video_level/train/traina[0-9].tfrecord .
```
Once you download the files, you can point the job to them using the
'train_data_pattern' argument (i.e. instead of pointing to the "gs://..."
files, you point to the local files).
Once your model is working locally, you can scale up on the Cloud
which is described below.
### Training on the Cloud over Video-Level Features
The following commands will train a model on Google Cloud
over video-level features.
```sh
BUCKET_NAME=gs://${USER}_yt8m_train_bucket
# (One Time) Create a storage bucket to store training logs and checkpoints.
gsutil mb -l us-east1 $BUCKET_NAME
# Submit the training job.
JOB_NAME=yt8m_train_$(date +%Y%m%d_%H%M%S); gcloud --verbosity=debug beta ml jobs \
submit training $JOB_NAME \
--package-path=youtube-8m --module-name=youtube-8m.train \
--staging-bucket=$BUCKET_NAME --region=us-east1 \
--config=youtube-8m/cloudml-gpu.yaml \
-- --train_data_pattern='gs://youtube8m-ml-us-east1/1/video_level/train/train*.tfrecord' \
--train_dir=$BUCKET_NAME/yt8m_train_video_level_logistic_model
```
In the 'gsutil' command above, the 'package-path' flag refers to the directory
containing the 'train.py' script and more generally the python package which
should be deployed to the cloud worker. The module-name refers to the specific
python script which should be executed (in this case the train module).
It may take several minutes before the job starts running on Google Cloud.
When it starts you will see outputs like the following:
```
training step 270| Hit@1: 0.68 PERR: 0.52 Loss: 638.453
training step 271| Hit@1: 0.66 PERR: 0.49 Loss: 635.537
training step 272| Hit@1: 0.70 PERR: 0.52 Loss: 637.564
```
At this point you can disconnect your console by pressing "ctrl-c". The
model will continue to train indefinitely in the Cloud. Later, you can check
on its progress or halt the job by visiting the
[Google Cloud ML Jobs console](https://console.cloud.google.com/ml/jobs).
You can train many jobs at once and use tensorboard to compare their performance
visually.
```sh
tensorboard --logdir=$BUCKET_NAME --port=8080
```
Once tensorboard is running, you can access it at the following url:
[http://localhost:8080](http://localhost:8080).
If you are using Google Cloud Shell, you can instead click the Web Preview button
on the upper left corner of the Cloud Shell window and select "Preview on port 8080".
This will bring up a new browser tab with the Tensorboard view.
### Evaluation and Inference
Here's how to evaluate a model on the validation dataset:
```sh
JOB_TO_EVAL=yt8m_train_video_level_logistic_model
JOB_NAME=yt8m_eval_$(date +%Y%m%d_%H%M%S); gcloud --verbosity=debug beta ml jobs \
submit training $JOB_NAME \
--package-path=youtube-8m --module-name=youtube-8m.eval \
--staging-bucket=$BUCKET_NAME --region=us-east1 \
--config=youtube-8m/cloudml-gpu.yaml \
-- --eval_data_pattern='gs://youtube8m-ml-us-east1/1/video_level/validate/validate*.tfrecord' \
--train_dir=$BUCKET_NAME/${JOB_TO_EVAL} --run_once=True
```
And here's how to perform inference with a model on the test set:
```sh
JOB_TO_EVAL=yt8m_train_video_level_logistic_model
JOB_NAME=yt8m_inference_$(date +%Y%m%d_%H%M%S); gcloud --verbosity=debug beta ml jobs \
submit training $JOB_NAME \
--package-path=youtube-8m --module-name=youtube-8m.inference \
--staging-bucket=$BUCKET_NAME --region=us-east1 \
--config=youtube-8m/cloudml-gpu.yaml \
-- --input_data_pattern='gs://youtube8m-ml/1/video_level/test/test*.tfrecord' \
--train_dir=$BUCKET_NAME/${JOB_TO_EVAL} \
--output_file=$BUCKET_NAME/${JOB_TO_EVAL}/predictions.csv
```
Note the confusing use of 'training' in the above gcloud commands. Despite the
name, the 'training' argument really just offers a cloud hosted
python/tensorflow service. From the point of view of the Cloud Platform, there
is no distinction between our training and inference jobs. The Cloud ML platform
also offers specialized functionality for prediction with
Tensorflow models, but discussing that is beyond the scope of this readme.
Once these job starts executing you will see outputs similar to the
following for the evaluation code:
```
examples_processed: 1024 | global_step 447044 | Batch Hit@1: 0.782 | Batch PERR: 0.637 | Batch Loss: 7.821 | Examples_per_sec: 834.658
```
and the following for the inference code:
```
num examples processed: 8192 elapsed seconds: 14.85
```
### Accessing Files on Google Cloud
You can browse the storage buckets you created on Google Cloud, for example, to
access the trained models, prediction CSV files, etc. by visiting the
[Google Cloud storage browser](https://console.cloud.google.com/storage/browser).
Alternatively, you can use the 'gsutil' command to download the files directly.
For example, to download the output of the inference code from the previous
section to your local machine, run:
```
gsutil cp $BUCKET_NAME/${JOB_TO_EVAL}/predictions.csv .
```
### Using Frame-Level Features
Append
```sh
--frame_features=True --model=FrameLevelLogisticModel --feature_names="rgb" \
--feature_sizes="1024" --batch_size=128 \
--train_dir=$BUCKET_NAME/yt8m_train_frame_level_logistic_model
```
to the 'gcloud' commands given above, and change 'video_level' in paths to
'frame_level'. Here is a sample command to kick-off a frame-level job:
```sh
JOB_NAME=yt8m_train_$(date +%Y%m%d_%H%M%S); gcloud --verbosity=debug beta ml jobs \
submit training $JOB_NAME \
--package-path=youtube-8m --module-name=youtube-8m.train \
--staging-bucket=$BUCKET_NAME --region=us-east1 \
--config=youtube-8m/cloudml-gpu.yaml \
-- --train_data_pattern='gs://youtube8m-ml-us-east1/1/frame_level/train/train*.tfrecord' \
--frame_features=True --model=FrameLevelLogisticModel --feature_names="rgb" \
--feature_sizes="1024" --batch_size=128 \
--train_dir=$BUCKET_NAME/yt8m_train_frame_level_logistic_model
```
The 'FrameLevelLogisticModel' is designed to provide equivalent results to a
logistic model trained over the video-level features. Please look at the
'video_level_models.py' or 'frame_level_models.py' files to see how to implement
your own models.
### Using Audio Features
The feature files (both Frame-Level and Video-Level) contain two sets of
features: 1) visual and 2) audio. The code defaults to using the visual
features only, but it is possible to use audio features instead of (or besides)
visual features. To specify the (combination of) features to use you must set
`--feature_names` and `--feature_sizes` flags. The visual and audio features are
called 'rgb' and 'audio' and have 1024 and 128 dimensions, respectively.
The two flags take a comma-separated list of values in string. For example, to
use audio-visual Video-Level features the flags must be set as follows:
```
--feature_names="mean_rgb, mean_audio" --feature_sizes="1024, 128"
```
Similarly, to use audio-visual Frame-Level features use:
```
--feature_names="rgb, audio" --feature_sizes="1024, 128"
```
**NOTE:** Make sure the set of features and the order in which the appear in the
lists provided to the two flags above match. Also, the order must match when
running training, evaluation, or inference.
## Running on Your Own Machine
### Requirements
The starter code requires Tensorflow. If you haven't installed it yet, follow
the instructions on [tensorflow.org](https://www.tensorflow.org/install/).
This code has been tested with Tensorflow 1.0.0. Going forward, we will continue
to target the latest released version of Tensorflow.
Please verify that you have Python 2.7+ and Tensorflow 1.0.0 or higher
installed by running the following commands:
```sh
python --version
python -c 'import tensorflow as tf; print(tf.__version__)'
```
You can find complete instructions for downloading the dataset on the
[YouTube-8M website](https://research.google.com/youtube8m/download.html).
We recommend downloading the smaller video-level features dataset first when
getting started. To do that, run:
```
mkdir -p features; cd features
curl data.yt8m.org/download.py | partition=1/video_level/train mirror=us python
```
This will download the full set of video level features, which takes up 31GB
of space.
If you are located outside of North America, you should change the flag 'mirror'
to 'eu' for Europe or 'asia' for Asia to speed up the transfer of the files.
Change 'train' to 'validate'/'test' and re-run the command to download the
other splits of the dataset.
Change 'video_level' to 'frame_level' to download the frame-level features. The
complete frame-level features take about 1.71TB of space. You can set the
environment variable 'shard' to 'm,n' to download only m/n-th of the data. For
example, to download 1/100-th of the frame-level features from the training set,
run:
```
curl data.yt8m.org/download.py | shard=1,100 partition=1/frame_level/train mirror=us python
```
### Training on Video-Level Features
To start training a logistic model on the video-level features, run
```sh
MODEL_DIR=/tmp/yt8m
python train.py --train_data_pattern='/path/to/features/train*.tfrecord' --train_dir=$MODEL_DIR/video_level_logistic_model
```
Since the dataset is sharded into 4096 individual files, we use a wildcard (\*)
to represent all of those files.
By default, the training code will frequently write _checkpoint_ files (i.e.
values of all trainable parameters, at the current training iteration). These
will be written to the `--train_dir`. If you re-use a `--train_dir`, the trainer
will first restore the latest checkpoint written in that directory. This only
works if the architecture of the checkpoint matches the graph created by the
training code. If you are in active development/debugging phase, consider
adding `--start_new_model` flag to your run configuration.
### Evaluation and Inference
To evaluate the model, run
```sh
python eval.py --eval_data_pattern='/path/to/features/validate*.tfrecord' --train_dir=$MODEL_DIR/video_level_logistic_model --run_once=True
```
As the model is training or evaluating, you can view the results on tensorboard
by running
```sh
tensorboard --logdir=$MODEL_DIR
```
and navigating to http://localhost:6006 in your web browser.
When you are happy with your model, you can generate a csv file of predictions
from it by running
```sh
python inference.py --output_file=$MODEL_DIR/video_level_logistic_model/predictions.csv --input_data_pattern='/path/to/features/test*.tfrecord' --train_dir=$MODEL_DIR/video_level_logistic_model
```
This will output the top 20 predicted labels from the model for every example
to 'predictions.csv'.
### Using Frame-Level Features
Follow the same instructions as above, appending
`--frame_features=True --model=FrameLevelLogisticModel --feature_names="rgb"
--feature_sizes="1024" --train_dir=$MODEL_DIR/frame_level_logistic_model`
for the 'train.py', 'eval.py', and 'inference.py' scripts.
The 'FrameLevelLogisticModel' is designed to provide equivalent results to a
logistic model trained over the video-level features. Please look at the
'models.py' file to see how to implement your own models.
### Using Audio Features
See [Using Audio Features](#using-audio-features) section above.
### Ground-Truth Label Files
We also provide CSV files containing the ground-truth label information of the
'train' and 'validation' partitions of the dataset. These files can be
downloaded using 'gsutil' command:
```
gsutil cp gs://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv /destination/folder/
gsutil cp gs://us.data.yt8m.org/1/ground_truth_labels/validate_labels.csv /destination/folder/
```
or directly using the following links:
* [http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv](http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv)
* [http://us.data.yt8m.org/1/ground_truth_labels/validate_labels.csv](http://us.data.yt8m.org/1/ground_truth_labels/validate_labels.csv)
Each line in the files starts with the video id and is followed by the list of
ground-truth labels corresponding to that video. For example, for a video with
id 'VIDEO_ID' and two lables 'LABLE1' and 'LABEL2' we store the following line:
```
VIDEO_ID,LABEL1 LABEL2
```
## Overview of Models
This sample code contains implementations of the models given in the
[YouTube-8M technical report](https://arxiv.org/abs/1609.08675).
### Video-Level Models
* `LogisticModel`: Linear projection of the output features into the label
space, followed by a sigmoid function to convert logit
values to probabilities.
* `MoeModel`: A per-class softmax distribution over a configurable number of
logistic classifiers. One of the classifiers in the mixture
is not trained, and always predicts 0.
### Frame-Level Models
* `LstmModel`: Processes the features for each frame using a multi-layered
LSTM neural net. The final internal state of the LSTM
is input to a video-level model for classification. Note that
you will need to change the learning rate to 0.001 when using
this model.
* `DbofModel`: Projects the features for each frame into a higher dimensional
'clustering' space, pools across frames in that space, and then
uses a video-level model to classify the now aggregated features.
* `FrameLevelLogisticModel`: Equivalent to 'LogisticModel', but performs
average-pooling on the fly over frame-level
features rather than using pre-aggregated features.
## Overview of Files
### Training
* `train.py`: The primary script for training models.
* `losses.py`: Contains definitions for loss functions.
* `models.py`: Contains the base class for defining a model.
* `video_level_models.py`: Contains definitions for models that take
aggregated features as input.
* `frame_level_models.py`: Contains definitions for models that take frame-
level features as input.
* `model_util.py`: Contains functions that are of general utility for
implementing models.
* `readers.py`: Contains definitions for the Video dataset and Frame
dataset readers.
### Evaluation
* `eval.py`: The primary script for evaluating models.
* `eval_util.py`: Provides a class that calculates all evaluation metrics.
* `average_precision_calculator.py`: Functions for calculating
average precision.
* `mean_average_precision_calculator.py`: Functions for calculating mean
average precision.
### Inference
* `inference.py`: Generates an output file containing predictions of
the model over a set of videos.
### Misc
* `README.md`: This documentation.
* `utils.py`: Common functions.
## About This Project
This project is meant help people quickly get started working with the
[YouTube-8M](https://research.google.com/youtube8m/) dataset.
This is not an official Google product.
================================================
FILE: youtube-8m-wangheda/__init__.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
================================================
FILE: youtube-8m-wangheda/all_data_augmentation/__init__.py
================================================
from default_augmenter import *
from half_augmenter import *
from half_video_augmenter import *
from noise_augmenter import *
from clipping_augmenter import *
================================================
FILE: youtube-8m-wangheda/all_data_augmentation/clipping_augmenter.py
================================================
import tensorflow as tf
from tensorflow import flags
FLAGS = flags.FLAGS
class ClippingAugmenter:
"""This only works with frame data"""
def augment(self, model_input_raw, num_frames, labels_batch, **unused_params):
assert(FLAGS.frame_feature,
"AugmentationTransformer only works with frame feature")
feature_dim = len(model_input_raw.get_shape()) - 1
frame_dim = len(model_input_raw.get_shape()) - 2
max_frame = model_input_raw.get_shape().as_list()[frame_dim]
limit = tf.cast(tf.reduce_min(num_frames) / 4.0, tf.int32)
offset = tf.random_uniform(shape=[], dtype=tf.int32) % limit
input_trans1 = tf.pad(model_input_raw[:,offset:,:], paddings=[0,offset,0])
num_frames_trans1 = num_frames - offset
num_frames_trans1 = tf.cast(
tf.random_uniform(shape=num_frames.shape, minval=0.75, maxval=1.0,
dtype=tf.float32)
* num_frames_trans1, tf.int32)
model_input = tf.concat([model_input_raw, input_trans1], axis=0)
labels_batch = tf.concat([labels_batch, labels_batch], axis=0)
num_frames = tf.concat([num_frames, num_frames_trans1], axis=0)
return model_input, labels_batch, num_frames_new
================================================
FILE: youtube-8m-wangheda/all_data_augmentation/default_augmenter.py
================================================
import tensorflow as tf
from tensorflow import flags
FLAGS = flags.FLAGS
class DefaultAugmenter:
"""This only works with frame data"""
def augment(self, model_input_raw, num_frames, labels_batch, **unused_params):
print "DefaultAugmenter"
return model_input_raw, labels_batch, num_frames
================================================
FILE: youtube-8m-wangheda/all_data_augmentation/half_augmenter.py
================================================
import tensorflow as tf
from tensorflow import flags
FLAGS = flags.FLAGS
class HalfAugmenter:
"""This only works with frame data"""
def augment(self, model_input_raw, num_frames, labels_batch, **unused_params):
assert(FLAGS.frame_features,
"HalfAugmenter only works with frame feature")
print "using HalfAugmeter"
feature_dim = len(model_input_raw.get_shape()) - 1
frame_dim = len(model_input_raw.get_shape()) - 2
max_frame = model_input_raw.get_shape().as_list()[frame_dim]
seg_length = max(int(max_frame / 2), 1)
seg_num_frames = tf.maximum(num_frames / 2, 1)
seg_inputs = []
seg_frames = []
seg_labels = []
seg_inputs.append(model_input_raw)
seg_frames.append(num_frames)
seg_labels.append(labels_batch)
for i in xrange(2):
begin_frames = tf.reshape(seg_num_frames*i, [-1,1])
frames_index = tf.reshape(tf.range(seg_length), [1,seg_length])
frames_index = begin_frames + frames_index
batch_size = tf.shape(model_input_raw)[0]
batch_index = tf.tile(tf.expand_dims(tf.range(batch_size), 1), [1, seg_length])
index = tf.stack([batch_index, tf.cast(frames_index,dtype=tf.int32)], 2)
seg_input = tf.gather_nd(model_input_raw, index)
seg_input = tf.pad(seg_input, paddings=[[0,0],[0, max_frame-seg_length],[0,0]])
seg_input = seg_input * tf.expand_dims(tf.sequence_mask(seg_num_frames, maxlen=max_frame, dtype=tf.float32), axis=2)
seg_inputs.append(seg_input)
seg_frames.append(seg_num_frames)
seg_labels.append(labels_batch)
new_input_raw = tf.concat(seg_inputs, axis=0)
new_num_frames = tf.concat(seg_frames, axis=0)
new_labels_batch = tf.concat(seg_labels, axis=0)
return new_input_raw, new_labels_batch, new_num_frames
================================================
FILE: youtube-8m-wangheda/all_data_augmentation/half_video_augmenter.py
================================================
import tensorflow as tf
from tensorflow import flags
FLAGS = flags.FLAGS
class HalfVideoAugmenter:
"""This only works with frame data"""
def augment(self, model_input_raw, num_frames, labels_batch, **unused_params):
assert(FLAGS.frame_features,
"HalfVideoAugmenter only works with frame feature")
print "using HalfVideoAugmeter"
new_input_raw, new_labels_batch, new_num_frames = self.frame_augment(model_input_raw, num_frames, labels_batch)
new_float_frames = tf.expand_dims(tf.cast(new_num_frames, tf.float32), axis=1)
aggregated_inputs = tf.reduce_sum(new_input_raw, axis=1) / new_float_frames
return aggregated_inputs, new_labels_batch, new_num_frames
def frame_augment(self, model_input_raw, num_frames, labels_batch, **unused_params):
feature_dim = len(model_input_raw.get_shape()) - 1
frame_dim = len(model_input_raw.get_shape()) - 2
max_frame = model_input_raw.get_shape().as_list()[frame_dim]
seg_length = max(int(max_frame / 2), 1)
seg_num_frames = tf.maximum(num_frames / 2, 1)
seg_inputs = []
seg_frames = []
seg_labels = []
seg_inputs.append(model_input_raw)
seg_frames.append(num_frames)
seg_labels.append(labels_batch)
for i in xrange(2):
begin_frames = tf.reshape(seg_num_frames*i, [-1,1])
frames_index = tf.reshape(tf.range(seg_length), [1,seg_length])
frames_index = begin_frames + frames_index
batch_size = tf.shape(model_input_raw)[0]
batch_index = tf.tile(tf.expand_dims(tf.range(batch_size), 1), [1, seg_length])
index = tf.stack([batch_index, tf.cast(frames_index,dtype=tf.int32)], 2)
seg_input = tf.gather_nd(model_input_raw, index)
seg_input = tf.pad(seg_input, paddings=[[0,0],[0, max_frame-seg_length],[0,0]])
seg_input = seg_input * tf.expand_dims(tf.sequence_mask(seg_num_frames, maxlen=max_frame, dtype=tf.float32), axis=2)
seg_inputs.append(seg_input)
seg_frames.append(seg_num_frames)
seg_labels.append(labels_batch)
new_input_raw = tf.concat(seg_inputs, axis=0)
new_num_frames = tf.concat(seg_frames, axis=0)
new_labels_batch = tf.concat(seg_labels, axis=0)
return new_input_raw, new_labels_batch, new_num_frames
================================================
FILE: youtube-8m-wangheda/all_data_augmentation/noise_augmenter.py
================================================
import tensorflow as tf
from tensorflow import flags
FLAGS = flags.FLAGS
class NoiseAugmenter:
"""This only works with frame data"""
def augment(self, model_input_raw, num_frames, labels_batch, **unused_params):
print "NoiseAugmenter", model_input_raw.shape, "noise =", FLAGS.input_noise_level
noise_input = tf.random_normal(tf.shape(model_input_raw), mean=0.0, stddev=FLAGS.input_noise_level)
model_input = model_input_raw + noise_input
return model_input, labels_batch, num_frames
================================================
FILE: youtube-8m-wangheda/all_feature_transform/__init__.py
================================================
from default_transformer import *
from identical_transformer import *
from engineer_transformer import *
from avg_transformer import *
from resolution_transformer import *
================================================
FILE: youtube-8m-wangheda/all_feature_transform/avg_transformer.py
================================================
import tensorflow as tf
class AvgTransformer:
def transform(self, model_input_raw, num_frames, **unused_params):
float_num_frames = tf.cast(tf.expand_dims(num_frames, 1), tf.float32)
feature_size = model_input_raw.get_shape().as_list()[2]
denominators = tf.reshape(
tf.tile(float_num_frames, [1, feature_size]), [-1, feature_size])
avg_pooled = tf.reduce_sum(model_input_raw, axis=[1]) / denominators
feature_dim = len(avg_pooled.get_shape()) - 1
model_input = tf.nn.l2_normalize(avg_pooled, feature_dim)
return model_input, num_frames
================================================
FILE: youtube-8m-wangheda/all_feature_transform/default_transformer.py
================================================
import tensorflow as tf
class DefaultTransformer:
def transform(self, model_input_raw, num_frames, **unused_params):
feature_dim = len(model_input_raw.get_shape()) - 1
model_input = tf.nn.l2_normalize(model_input_raw, feature_dim)
return model_input, num_frames
================================================
FILE: youtube-8m-wangheda/all_feature_transform/engineer_transformer.py
================================================
import tensorflow as tf
import numpy as np
from tensorflow import flags
FLAGS = flags.FLAGS
class EngineerTransformer:
"""feature transform by feature engineering"""
def transform(self, model_input_raw, num_frames, **unused_params):
feature_dim = len(model_input_raw.get_shape()) - 1
engineer_types = map(lambda x: x.strip(), FLAGS.engineer_types.split(","))
feature_list = []
mask = self.mask(model_input_raw, num_frames)
for etype in engineer_types:
if etype == "avg":
feature_list.append(self.avg(model_input_raw, num_frames, mask))
elif etype == "std":
feature_list.append(self.std(model_input_raw, num_frames, mask))
elif etype == "diff":
feature_list.append(self.diff(model_input_raw, num_frames, mask))
else:
feature_list.append(model_input_raw)
model_input = tf.nn.l2_normalize(tf.concat(model_input_raw, axis=feature_dim), feature_dim)
return model_input, num_frames
def mask(self, model_input_raw, num_frames):
max_frames = model_input_raw.get_shape().as_list()[1]
mask_array = []
for i in xrange(max_frames + 1):
tmp = [0.0] * max_frames
for j in xrange(i):
tmp[j] = 1.0
mask_array.append(tmp)
mask_array = np.array(mask_array)
mask_init = tf.constant_initializer(mask_array)
mask_emb = tf.get_variable("mask_emb", shape = [max_frames + 1, max_frames],
dtype = tf.float32, trainable = False, initializer = mask_init)
mask = tf.nn.embedding_lookup(mask_emb, num_frames)
return mask
def avg(self, model_input_raw, num_frames, mask):
max_frames = model_input_raw.get_shape().as_list()[1]
num_frames_matrix = tf.maximum(tf.cast(
tf.expand_dims(num_frames, axis=1),
dtype=tf.float32), 1.0)
mean_matrix = mask / num_frames_matrix
mean_input = tf.einsum("ijk,ij->ik", model_input_raw, mean_matrix)
mean_input_tile = tf.tile(tf.expand_dims(mean_input, axis=1), multiples=[1,max_frames,1])
return mean_input_tile
def std(self, model_input_raw, num_frames, mask):
mean_input = self.avg(model_input_raw, num_frames, mask)
error = tf.einsum("ijk,ij->ijk", model_input_raw - mean_input, mask)
return error
def diff(self, model_input_raw, num_frames, mask):
max_frames = model_input_raw.get_shape().as_list()[1]
shift_input1 = tf.pad(model_input_raw, paddings=[[0,0], [0,1], [0,0]])
shift_input2 = tf.pad(model_input_raw, paddings=[[0,0], [1,0], [0,0]])
diff_input = shift_input1 - shift_input2
difference = tf.einsum("ijk,ij->ijk", diff_input[:,:max_frames,:], mask)
return difference
================================================
FILE: youtube-8m-wangheda/all_feature_transform/identical_transformer.py
================================================
import tensorflow as tf
class IdenticalTransformer:
def transform(self, model_input_raw, num_frames, **unused_params):
return model_input_raw, num_frames
================================================
FILE: youtube-8m-wangheda/all_feature_transform/resolution_transformer.py
================================================
import tensorflow as tf
from tensorflow import flags
FLAGS = flags.FLAGS
class ResolutionTransformer:
def resolution(self, model_input_raw, num_frames):
resolution = FLAGS.time_resolution
frame_dim = len(model_input_raw.get_shape()) - 2
feature_dim = len(model_input_raw.get_shape()) - 1
max_frames = model_input_raw.get_shape().as_list()[frame_dim]
num_features = model_input_raw.get_shape().as_list()[feature_dim]
new_max_frames = max_frames / resolution
cut_frames = new_max_frames * resolution
model_input_raw = model_input_raw[:, :cut_frames, :]
model_input_raw = tf.reshape(model_input_raw, shape=[-1,new_max_frames,resolution,num_features])
model_input_raw = tf.reduce_mean(model_input_raw, axis=2)
num_frames = num_frames / resolution
return model_input_raw, num_frames
def transform(self, model_input_raw, num_frames, **unused_params):
model_input_raw, num_frames = self.resolution(model_input_raw, num_frames)
feature_dim = len(model_input_raw.get_shape()) - 1
model_input = tf.nn.l2_normalize(model_input_raw, feature_dim)
return model_input, num_frames
================================================
FILE: youtube-8m-wangheda/all_frame_models/.vimrc
================================================
syntax on
set tabstop=2
set shiftwidth=2
set expandtab
set autoindent
================================================
FILE: youtube-8m-wangheda/all_frame_models/__init__.py
================================================
from bilstm_model import *
from biunilstm_model import *
from cnn_kmax_model import *
from dbof_model import *
from frame_seg_model import *
from deep_lstm_model import *
from gru_pooling_model import *
from gru_with_pooling_model import *
from logistic_model import *
from lstm_advanced_model import *
from lstm_attention_lstm_model import *
from lstm_attention_model import *
from lstm_attention_max_pooling_model import *
from distillchain_lstm_attention_max_pooling_model import *
from lstm_positional_attention_max_pooling_model import *
from lstm_divided_model import *
from lstm_memory_model import *
from layernorm_lstm_memory_model import *
from lstm_parallel_memory_model import *
from lstm_parallel_finaloutput_model import *
from distillchain_lstm_parallel_finaloutput_model import *
from cnn_model import *
from cnn_deep_combine_chain_model import *
from distillchain_cnn_deep_combine_chain_model import *
from positional_cnn_deep_combine_chain_model import *
from multi_view_cnn_deep_combine_chain_model import *
from lstm_cnn_deep_combine_chain_model import *
from distillchain_lstm_cnn_deep_combine_chain_model import *
from deep_cnn_deep_combine_chain_model import *
from cnn_lstm_memory_model import *
from cnn_lstm_memory_multitask_model import *
from cnn_lstm_memory_normalization_model import *
from mm_lstm_memory_model import *
from lstm_memory_normalization_model import *
from lstm_model import *
from lstm_multi_attention_model import *
from lstm_multi_pooling_model import *
from lstm_parallel_model import *
from lstm_pooling_model import *
from lstm_with_mean_input_model import *
from lstm_with_pooling_model import *
from progressive_attention_lstm_model import *
from lstm_memory_multitask_model import *
from lstm_memory_chain_model import *
from lstm_memory_deep_chain_model import *
from distillchain_lstm_memory_deep_combine_chain_model import *
from lstm_memory_input_chain_model import *
from lstm_memory_parallel_chain_model import *
from wide_and_deep_model import *
from multires_lstm_memory_deep_combine_chain_model import *
from framehop_lstm_memory_deep_combine_chain_model import *
from framehop_lstm_memory_model import *
from lstm_look_back_model import *
from lstm_auxloss_deep_combine_chain_model import *
from multiscale_cnn_lstm_model import *
from distillchain_multiscale_cnn_lstm_model import *
================================================
FILE: youtube-8m-wangheda/all_frame_models/bilstm_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class BiLstmModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of Bi LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
## Batch normalize the input
fw_stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=False)
for _ in range(number_of_layers)
],
state_is_tuple=False)
bw_stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=False)
for _ in range(number_of_layers)
],
state_is_tuple=False)
loss = 0.0
with tf.variable_scope("RNN"):
outputs, states = tf.nn.bidirectional_dynamic_rnn(cell_fw = fw_stacked_lstm, cell_bw = bw_stacked_lstm,
inputs = model_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
state_fw, state_bw = states
state = tf.concat([state_fw, state_bw], axis = 1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=state,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
================================================
FILE: youtube-8m-wangheda/all_frame_models/biunilstm_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class BiUniLstmModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of Bi-Uni LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
## Batch normalize the input
fw_cell = tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=False)
bw_cell = tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=False)
cell = tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=False)
loss = 0.0
with tf.variable_scope("RNN"):
l1_outputs, l1_states = tf.nn.bidirectional_dynamic_rnn(cell_fw = fw_cell, cell_bw = bw_cell,
inputs = model_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
l1_outputs = tf.concat(l1_outputs, axis = 2)
l2_outputs, l2_states = tf.nn.dynamic_rnn(cell=cell,
inputs=l1_outputs,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
state_fw, state_bw = l1_states
state = tf.concat([state_fw, state_bw, l2_states], axis = 1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=state,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
================================================
FILE: youtube-8m-wangheda/all_frame_models/cnn_deep_combine_chain_model.py
================================================
import math
import models
import tensorflow as tf
import numpy as np
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class CnnDeepCombineChainModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def cnn(self,
model_input,
l2_penalty=1e-8,
num_filters = [1024, 1024, 1024],
filter_sizes = [1,2,3],
sub_scope="",
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
shift_inputs = []
for i in xrange(max(filter_sizes)):
if i == 0:
shift_inputs.append(model_input)
else:
shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:])
cnn_outputs = []
for nf, fs in zip(num_filters, filter_sizes):
sub_input = tf.concat(shift_inputs[:fs], axis=2)
sub_filter = tf.get_variable(sub_scope+"cnn-filter-len%d"%fs,
shape=[num_features*fs, nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter))
cnn_output = tf.concat(cnn_outputs, axis=2)
return cnn_output
def create_model(self, model_input, vocab_size, num_frames, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", original_input=None, **unused_params):
num_supports = FLAGS.num_supports
num_layers = FLAGS.deep_chain_layers
relu_cells = FLAGS.deep_chain_relu_cells
max_frames = model_input.get_shape().as_list()[1]
relu_layers = []
support_predictions = []
mask = self.get_mask(max_frames, num_frames)
mean_input = tf.einsum("ijk,ij->ik", model_input, mask) \
/ tf.expand_dims(tf.cast(num_frames, dtype=tf.float32), dim=1)
mean_relu = slim.fully_connected(
mean_input,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"mean-relu")
mean_relu_norm = tf.nn.l2_normalize(mean_relu, dim=1)
relu_layers.append(mean_relu_norm)
cnn_output = self.cnn(model_input, num_filters=[relu_cells,relu_cells,relu_cells*2], filter_sizes=[1,2,3], sub_scope=sub_scope+"cnn0")
max_cnn_output = tf.reduce_max(cnn_output, axis=1)
normalized_cnn_output = tf.nn.l2_normalize(max_cnn_output, dim=1)
next_input = normalized_cnn_output
for layer in xrange(num_layers):
sub_prediction = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"prediction-%d"%layer)
support_predictions.append(sub_prediction)
sub_relu = slim.fully_connected(
sub_prediction,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"relu-%d"%layer)
relu_norm = tf.nn.l2_normalize(sub_relu, dim=1)
relu_layers.append(relu_norm)
cnn_output = self.cnn(model_input, num_filters=[relu_cells,relu_cells,relu_cells*2], filter_sizes=[1,2,3], sub_scope=sub_scope+"cnn%d"%(layer+1))
max_cnn_output = tf.reduce_max(cnn_output, axis=1)
normalized_cnn_output = tf.nn.l2_normalize(max_cnn_output, dim=1)
next_input = tf.concat([mean_input, normalized_cnn_output] + relu_layers, axis=1)
main_predictions = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"-main")
support_predictions = tf.concat(support_predictions, axis=1)
return {"predictions": main_predictions, "support_predictions": support_predictions}
def sub_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", **unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
def get_mask(self, max_frames, num_frames):
mask_array = []
for i in xrange(max_frames + 1):
tmp = [0.0] * max_frames
for j in xrange(i):
tmp[j] = 1.0
mask_array.append(tmp)
mask_array = np.array(mask_array)
mask_init = tf.constant_initializer(mask_array)
mask_emb = tf.get_variable("mask_emb", shape = [max_frames + 1, max_frames],
dtype = tf.float32, trainable = False, initializer = mask_init)
mask = tf.nn.embedding_lookup(mask_emb, num_frames)
return mask
================================================
FILE: youtube-8m-wangheda/all_frame_models/cnn_kmax_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class CnnKmaxModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
# Create a convolution + maxpool layer for each filter size
filter_sizes = map(int, FLAGS.cnn_filter_sizes.split(","))
filter_nums = map(int, FLAGS.cnn_filter_nums.split(","))
pooling_k = FLAGS.cnn_pooling_k
assert len(filter_sizes) == len(filter_nums), \
"length of filter_sizes (={}) != length of filter_nums (={})".format( \
len(filter_sizes), len(filter_nums))
batch_size, max_frames, num_features = model_input.get_shape().as_list()
with tf.variable_scope("CNN"):
# set channel dimension to 1
# cnn_input.shape = [batch, in_height, in_width, in_channels]
cnn_input = tf.expand_dims(model_input, axis = 3)
cnn_output = []
for filter_size, filter_num in zip(filter_sizes, filter_nums):
with tf.name_scope("conv-maxpool-%s" % filter_size):
# Convolution Layer
# filter.shape = [filter_height, filter_width, in_channels, out_channels]
filter_shape = [filter_size, num_features, 1, filter_num]
W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name="W")
b = tf.Variable(tf.constant(0.1, shape=[filter_num]), name="b")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b))
# conv.shape = [batch, -1, 1, out_channels]
conv = tf.nn.conv2d(cnn_input, W, strides=[1, 1, 1, 1], padding="VALID",name="conv")
# add bias
conv = tf.nn.bias_add(conv, b)
if pooling_k == 1:
_, conv_len, _, _= conv.shape.as_list()
conv_kmax = tf.reduce_max(conv, axis = 1)
conv_flat = tf.reshape(conv_kmax, [-1, filter_num])
else:
conv_kmax = tf.transpose(tf.squeeze(conv, axis = 2), perm = [0, 2, 1])
conv_kmax, _ = tf.nn.top_k(conv_kmax, k = pooling_k, sorted = True)
conv_flat = tf.reshape(conv_kmax, [-1, filter_num * pooling_k])
cnn_output.append(conv_flat)
cnn_output = tf.concat(cnn_output, axis = 1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=conv_flat,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
================================================
FILE: youtube-8m-wangheda/all_frame_models/cnn_lstm_memory_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class CnnLstmMemoryModel(models.BaseModel):
def cnn(self,
model_input,
l2_penalty=1e-8,
num_filters = [1024, 1024, 1024],
filter_sizes = [1,2,3],
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
shift_inputs = []
for i in xrange(max(filter_sizes)):
if i == 0:
shift_inputs.append(model_input)
else:
shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:])
cnn_outputs = []
for nf, fs in zip(num_filters, filter_sizes):
sub_input = tf.concat(shift_inputs[:fs], axis=2)
sub_filter = tf.get_variable("cnn-filter-len%d"%fs, shape=[num_features*fs, nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter))
cnn_output = tf.concat(cnn_outputs, axis=2)
return cnn_output
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
cnn_output = self.cnn(model_input, num_filters=[1024,1024,1024], filter_sizes=[1,2,3])
normalized_cnn_output = tf.nn.l2_normalize(cnn_output, dim=2)
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
loss = 0.0
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, normalized_cnn_output,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
final_state = tf.concat(map(lambda x: x.c, state), axis = 1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=final_state,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
================================================
FILE: youtube-8m-wangheda/all_frame_models/cnn_lstm_memory_multitask_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class CnnLstmMemoryMultiTaskModel(models.BaseModel):
def cnn(self,
model_input,
l2_penalty=1e-8,
num_filters = [1024, 1024, 1024],
filter_sizes = [1,2,3],
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
shift_inputs = []
for i in xrange(max(filter_sizes)):
if i == 0:
shift_inputs.append(model_input)
else:
shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:])
cnn_outputs = []
for nf, fs in zip(num_filters, filter_sizes):
sub_input = tf.concat(shift_inputs[:fs], axis=2)
sub_filter = tf.get_variable("cnn-filter-len%d"%fs, shape=[num_features*fs, nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter))
cnn_output = tf.concat(cnn_outputs, axis=2)
return cnn_output
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
max_frames = model_input.get_shape().as_list()[1]
cnn_output = self.cnn(model_input, num_filters=[1024,1024,1024], filter_sizes=[1,2,3])
normalized_cnn_output = tf.nn.l2_normalize(cnn_output, dim=2)
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
loss = 0.0
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, normalized_cnn_output,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
final_state = tf.concat(map(lambda x: x.c, state), axis = 1)
mask = self.get_mask(max_frames, num_frames)
mean_cnn_output = tf.einsum("ijk,ij->ik", normalized_cnn_output, mask) \
/ tf.expand_dims(tf.cast(num_frames, dtype=tf.float32), dim=1)
support_model = getattr(video_level_models,
FLAGS.video_level_classifier_support_model)
support_predictions = support_model().create_model(
model_input=mean_cnn_output,
original_input=model_input,
vocab_size=vocab_size,
num_mixtures=2,
sub_scope="support",
**unused_params)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
predictions = aggregated_model().create_model(
model_input=final_state,
original_input=model_input,
vocab_size=vocab_size,
sub_scope="main",
**unused_params)
return {"predictions": predictions["predictions"],
"support_predictions": support_predictions["predictions"]}
def get_mask(self, max_frames, num_frames):
mask_array = []
for i in xrange(max_frames + 1):
tmp = [0.0] * max_frames
for j in xrange(i):
tmp[j] = 1.0
mask_array.append(tmp)
mask_array = np.array(mask_array)
mask_init = tf.constant_initializer(mask_array)
mask_emb = tf.get_variable("mask_emb", shape = [max_frames + 1, max_frames],
dtype = tf.float32, trainable = False, initializer = mask_init)
mask = tf.nn.embedding_lookup(mask_emb, num_frames)
return mask
================================================
FILE: youtube-8m-wangheda/all_frame_models/cnn_lstm_memory_normalization_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class CnnLstmMemoryNormalizationModel(models.BaseModel):
def cnn(self,
model_input,
l2_penalty=1e-8,
num_filters = [1024, 1024, 1024],
filter_sizes = [1,2,3],
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
normalize_class = getattr(self, FLAGS.lstm_normalization, self.identical)
shift_inputs = []
for i in xrange(max(filter_sizes)):
if i == 0:
shift_inputs.append(model_input)
else:
shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:])
cnn_outputs = []
for nf, fs in zip(num_filters, filter_sizes):
sub_input = tf.concat(shift_inputs[:fs], axis=2)
sub_filter = tf.get_variable("cnn-filter-len%d"%fs, shape=[num_features*fs, nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter))
cnn_output = tf.concat(cnn_outputs, axis=2)
return cnn_output
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
cnn_output = self.cnn(model_input, num_filters=[1024,1024,1024], filter_sizes=[1,2,3])
outputs, state = cnn_output, []
for layer in xrange(number_of_layers):
with tf.variable_scope("RNN-layer%d" % layer):
outputs = normalize_class(outputs)
cell = tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
layer_outputs, layer_state = tf.nn.dynamic_rnn(cell, outputs,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
state.append(layer_state)
outputs = layer_outputs
final_state = tf.concat(map(lambda x: x.c, state), axis = 1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=final_state,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
def layer_normalize(self, input_raw, epsilon=1e-8):
feature_dim = len(input_raw.get_shape()) - 1
mean_input = tf.reduce_mean(input_raw, axis=feature_dim, keep_dims=True)
std_input = tf.sqrt(tf.reduce_mean(tf.square(input_raw-mean_input), axis=feature_dim, keep_dims=True))
std_input = tf.maximum(std_input, epsilon)
output = (input_raw - mean_input) / std_input
return output
def l2_normalize(self, input_raw, epsilon=1e-8):
feature_dim = len(input_raw.get_shape()) - 1
output = tf.nn.l2_normalize(input_raw, dim=feature_dim)
return output
def identical(self, input_raw, epsilon=1e-8):
return input_raw
================================================
FILE: youtube-8m-wangheda/all_frame_models/cnn_model.py
================================================
import math
import models
import tensorflow as tf
import numpy as np
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class CnnModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def cnn(self,
model_input,
l2_penalty=1e-8,
num_filters = [1024, 1024, 1024],
filter_sizes = [1,2,3],
sub_scope="",
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
shift_inputs = []
for i in xrange(max(filter_sizes)):
if i == 0:
shift_inputs.append(model_input)
else:
shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:])
cnn_outputs = []
for nf, fs in zip(num_filters, filter_sizes):
sub_input = tf.concat(shift_inputs[:fs], axis=2)
sub_filter = tf.get_variable(sub_scope+"cnn-filter-len%d"%fs,
shape=[num_features*fs, nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter))
cnn_output = tf.concat(cnn_outputs, axis=2)
return cnn_output
def create_model(self, model_input, vocab_size, num_frames, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", original_input=None, **unused_params):
num_supports = FLAGS.num_supports
num_filters = FLAGS.cnn_num_filters
max_frames = model_input.get_shape().as_list()[1]
cnn_output = self.cnn(model_input, num_filters=[num_filters,num_filters,num_filters*2], filter_sizes=[1,2,3], sub_scope=sub_scope+"cnn")
max_cnn_output = tf.reduce_max(cnn_output, axis=1)
normalized_cnn_output = tf.nn.l2_normalize(max_cnn_output, dim=1)
predictions = self.sub_model(normalized_cnn_output, vocab_size, sub_scope=sub_scope+"main")
return {"predictions": predictions}
def sub_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", **unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
================================================
FILE: youtube-8m-wangheda/all_frame_models/dbof_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class DbofModel(models.BaseModel):
"""Creates a Deep Bag of Frames model.
The model projects the features for each frame into a higher dimensional
'clustering' space, pools across frames in that space, and then
uses a configurable video-level model to classify the now aggregated features.
The model will randomly sample either frames or sequences of frames during
training to speed up convergence.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
def create_model(self,
model_input,
vocab_size,
num_frames,
iterations=None,
add_batch_norm=None,
sample_random_frames=None,
cluster_size=None,
hidden_size=None,
is_training=True,
**unused_params):
iterations = iterations or FLAGS.iterations
add_batch_norm = add_batch_norm or FLAGS.dbof_add_batch_norm
random_frames = sample_random_frames or FLAGS.sample_random_frames
cluster_size = cluster_size or FLAGS.dbof_cluster_size
hidden1_size = hidden_size or FLAGS.dbof_hidden_size
num_frames = tf.cast(tf.expand_dims(num_frames, 1), tf.float32)
if random_frames:
model_input = model_utils.SampleRandomFrames(model_input, num_frames,
iterations)
else:
model_input = model_utils.SampleRandomSequence(model_input, num_frames,
iterations)
max_frames = model_input.get_shape().as_list()[1]
feature_size = model_input.get_shape().as_list()[2]
reshaped_input = tf.reshape(model_input, [-1, feature_size])
tf.summary.histogram("input_hist", reshaped_input)
if add_batch_norm:
reshaped_input = slim.batch_norm(
reshaped_input,
center=True,
scale=True,
is_training=is_training,
scope="input_bn")
cluster_weights = tf.Variable(tf.random_normal(
[feature_size, cluster_size],
stddev=1 / math.sqrt(feature_size)))
tf.summary.histogram("cluster_weights", cluster_weights)
activation = tf.matmul(reshaped_input, cluster_weights)
if add_batch_norm:
activation = slim.batch_norm(
activation,
center=True,
scale=True,
is_training=is_training,
scope="cluster_bn")
else:
cluster_biases = tf.Variable(
tf.random_normal(
[cluster_size], stddev=1 / math.sqrt(feature_size)))
tf.summary.histogram("cluster_biases", cluster_biases)
activation += cluster_biases
activation = tf.nn.relu6(activation)
tf.summary.histogram("cluster_output", activation)
activation = tf.reshape(activation, [-1, max_frames, cluster_size])
activation = model_utils.FramePooling(activation, FLAGS.dbof_pooling_method)
hidden1_weights = tf.Variable(tf.random_normal(
[cluster_size, hidden1_size],
stddev=1 / math.sqrt(cluster_size)))
tf.summary.histogram("hidden1_weights", hidden1_weights)
activation = tf.matmul(activation, hidden1_weights)
if add_batch_norm:
activation = slim.batch_norm(
activation,
center=True,
scale=True,
is_training=is_training,
scope="hidden1_bn")
else:
hidden1_biases = tf.Variable(
tf.random_normal(
[hidden1_size], stddev=0.01))
tf.summary.histogram("hidden1_biases", hidden1_biases)
activation += hidden1_biases
activation = tf.nn.relu6(activation)
tf.summary.histogram("hidden1_output", activation)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=activation,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
================================================
FILE: youtube-8m-wangheda/all_frame_models/deep_cnn_deep_combine_chain_model.py
================================================
import math
import models
import tensorflow as tf
import numpy as np
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class DeepCnnDeepCombineChainModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def cnn(self,
model_input,
l2_penalty=1e-8,
num_filters = [128,128,256],
filter_sizes = [1,2,3],
sub_scope="",
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
shift_inputs = []
for i in xrange(max(filter_sizes)):
if i == 0:
shift_inputs.append(model_input)
else:
shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:])
cnn_outputs = []
for nf, fs in zip(num_filters, filter_sizes):
sub_input = tf.concat(shift_inputs[:fs], axis=2)
sub_filter = tf.get_variable(sub_scope+"filter-len%d"%fs,
shape=[num_features*fs, nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter))
cnn_output = tf.concat(cnn_outputs, axis=2)
return cnn_output
def deep_cnn(self,
model_input,
l2_penalty=1e-8,
num_layers=3,
num_filters = [[128,128,256],[128,128],[128,128]],
filter_sizes = [[1,2,3],[2,3],[2,3]],
sub_scope="",
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
cnn_outputs = []
time_pooled = model_input
for i in xrange(num_layers):
cnn_output = self.cnn(time_pooled, num_filters=num_filters[i],
filter_sizes=filter_sizes[i], sub_scope=sub_scope+"cnn%d"%i)
cnn_outputs.append(cnn_output)
if i+1 < num_layers:
time_pooled = tf.nn.pool(cnn_output, window_shape=[2],
pooling_type="MAX", padding="VALID", strides=[2])
return cnn_outputs
def create_model(self, model_input, vocab_size, num_frames, num_mixtures=None,
dropout=False, keep_prob=None, noise_level=None,
l2_penalty=1e-8, sub_scope="", original_input=None, **unused_params):
num_supports = FLAGS.num_supports
num_layers = FLAGS.deep_chain_layers
relu_cells = FLAGS.deep_chain_relu_cells
cnn_size = FLAGS.deep_cnn_base_size
max_frames = model_input.get_shape().as_list()[1]
relu_layers = []
support_predictions = []
mask = self.get_mask(max_frames, num_frames)
mean_input = tf.einsum("ijk,ij->ik", model_input, mask) \
/ tf.expand_dims(tf.cast(num_frames, dtype=tf.float32), dim=1)
mean_relu = slim.fully_connected(
mean_input,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"mean-relu")
mean_relu_norm = tf.nn.l2_normalize(mean_relu, dim=1)
relu_layers.append(mean_relu_norm)
cnn_outputs = self.deep_cnn(model_input, sub_scope=sub_scope+"deepcnn0",
num_layers=3,
num_filters = [[cnn_size,cnn_size,cnn_size*2],[cnn_size,cnn_size],[cnn_size,cnn_size]],
filter_sizes = [[1,2,3],[2,3],[2,3]])
max_cnn_output = tf.concat(map(lambda x: tf.reduce_max(x, axis=1), cnn_outputs), axis=1)
normalized_cnn_output = tf.nn.l2_normalize(max_cnn_output, dim=1)
next_input = normalized_cnn_output
for layer in xrange(num_layers):
sub_prediction = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"prediction-%d"%layer,
dropout=dropout, keep_prob=keep_prob, noise_level=noise_level)
support_predictions.append(sub_prediction)
sub_relu = slim.fully_connected(
sub_prediction,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"relu-%d"%layer)
relu_norm = tf.nn.l2_normalize(sub_relu, dim=1)
relu_layers.append(relu_norm)
cnn_outputs = self.deep_cnn(model_input, sub_scope=sub_scope+"deepcnn%d"%(layer+1),
num_layers=3,
num_filters = [[cnn_size,cnn_size,cnn_size*2],[cnn_size,cnn_size],[cnn_size,cnn_size]],
filter_sizes = [[1,2,3],[2,3],[2,3]])
max_cnn_output = tf.concat(map(lambda x: tf.reduce_max(x, axis=1), cnn_outputs), axis=1)
normalized_cnn_output = tf.nn.l2_normalize(max_cnn_output, dim=1)
next_input = tf.concat([mean_input, normalized_cnn_output] + relu_layers, axis=1)
main_predictions = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"-main",
dropout=dropout, keep_prob=keep_prob, noise_level=noise_level)
support_predictions = tf.concat(support_predictions, axis=1)
return {"predictions": main_predictions, "support_predictions": support_predictions}
def sub_model(self, model_input, vocab_size, num_mixtures=None,
dropout=False, keep_prob=None, noise_level=None,
l2_penalty=1e-8, sub_scope="", **unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
if dropout:
print "adding dropout to", model_input
model_input = tf.nn.dropout(model_input, keep_prob=keep_prob)
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
def get_mask(self, max_frames, num_frames):
mask_array = []
for i in xrange(max_frames + 1):
tmp = [0.0] * max_frames
for j in xrange(i):
tmp[j] = 1.0
mask_array.append(tmp)
mask_array = np.array(mask_array)
mask_init = tf.constant_initializer(mask_array)
mask_emb = tf.get_variable("mask_emb", shape = [max_frames + 1, max_frames],
dtype = tf.float32, trainable = False, initializer = mask_init)
mask = tf.nn.embedding_lookup(mask_emb, num_frames)
return mask
================================================
FILE: youtube-8m-wangheda/all_frame_models/deep_lstm_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class DeepLstmModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size / (2 ** layer_ind), forget_bias=1.0, state_is_tuple=False)
for layer_ind in range(number_of_layers)
],
state_is_tuple=False)
loss = 0.0
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
time_major=False,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
concat_state = tf.concat(state, axis = 1)
num_frames_matrix = tf.maximum(tf.cast(tf.expand_dims(num_frames, axis=1), dtype=tf.float32), 1.0)
pooling_output = tf.reduce_sum(outputs, axis = 1) / num_frames_matrix
final_output = tf.concat([pooling_output, concat_state], axis = 1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
predictions = aggregated_model().create_model(
model_input=final_output,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
return predictions
================================================
FILE: youtube-8m-wangheda/all_frame_models/distillchain_cnn_deep_combine_chain_model.py
================================================
import math
import models
import tensorflow as tf
import numpy as np
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class DistillchainCnnDeepCombineChainModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def cnn(self,
model_input,
l2_penalty=1e-8,
num_filters = [1024, 1024, 1024],
filter_sizes = [1,2,3],
sub_scope="",
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
shift_inputs = []
for i in xrange(max(filter_sizes)):
if i == 0:
shift_inputs.append(model_input)
else:
shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:])
cnn_outputs = []
for nf, fs in zip(num_filters, filter_sizes):
sub_input = tf.concat(shift_inputs[:fs], axis=2)
sub_filter = tf.get_variable(sub_scope+"cnn-filter-len%d"%fs,
shape=[num_features*fs, nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter))
cnn_output = tf.concat(cnn_outputs, axis=2)
return cnn_output
def create_model(self, model_input, vocab_size, num_frames, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", original_input=None,
distillation_predictions=None,
**unused_params):
assert distillation_predictions is not None, "distillation feature must be used"
distillchain_relu_cells = FLAGS.distillchain_relu_cells
num_supports = FLAGS.num_supports
num_layers = FLAGS.deep_chain_layers
relu_cells = FLAGS.deep_chain_relu_cells
max_frames = model_input.get_shape().as_list()[1]
relu_layers = []
support_predictions = []
distill_relu = slim.fully_connected(
distillation_predictions,
distillchain_relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="distillrelu")
distill_norm = tf.nn.l2_normalize(distill_relu, dim=1)
relu_layers.append(distill_norm)
mask = self.get_mask(max_frames, num_frames)
mean_input = tf.einsum("ijk,ij->ik", model_input, mask) \
/ tf.expand_dims(tf.cast(num_frames, dtype=tf.float32), dim=1)
mean_relu = slim.fully_connected(
mean_input,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"mean-relu")
mean_relu_norm = tf.nn.l2_normalize(mean_relu, dim=1)
relu_layers.append(mean_relu_norm)
cnn_output = self.cnn(model_input, num_filters=[relu_cells,relu_cells,relu_cells*2], filter_sizes=[1,2,3], sub_scope=sub_scope+"cnn0")
max_cnn_output = tf.reduce_max(cnn_output, axis=1)
normalized_cnn_output = tf.nn.l2_normalize(max_cnn_output, dim=1)
next_input = tf.concat([normalized_cnn_output] + relu_layers, axis=1)
for layer in xrange(num_layers):
sub_prediction = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"prediction-%d"%layer)
support_predictions.append(sub_prediction)
sub_relu = slim.fully_connected(
sub_prediction,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"relu-%d"%layer)
relu_norm = tf.nn.l2_normalize(sub_relu, dim=1)
relu_layers.append(relu_norm)
cnn_output = self.cnn(model_input, num_filters=[relu_cells,relu_cells,relu_cells*2], filter_sizes=[1,2,3], sub_scope=sub_scope+"cnn%d"%(layer+1))
max_cnn_output = tf.reduce_max(cnn_output, axis=1)
normalized_cnn_output = tf.nn.l2_normalize(max_cnn_output, dim=1)
next_input = tf.concat([normalized_cnn_output] + relu_layers, axis=1)
main_predictions = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"-main")
support_predictions = tf.concat(support_predictions, axis=1)
return {"predictions": main_predictions, "support_predictions": support_predictions}
def sub_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", **unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
def get_mask(self, max_frames, num_frames):
mask_array = []
for i in xrange(max_frames + 1):
tmp = [0.0] * max_frames
for j in xrange(i):
tmp[j] = 1.0
mask_array.append(tmp)
mask_array = np.array(mask_array)
mask_init = tf.constant_initializer(mask_array)
mask_emb = tf.get_variable("mask_emb", shape = [max_frames + 1, max_frames],
dtype = tf.float32, trainable = False, initializer = mask_init)
mask = tf.nn.embedding_lookup(mask_emb, num_frames)
return mask
================================================
FILE: youtube-8m-wangheda/all_frame_models/distillchain_lstm_attention_max_pooling_model.py
================================================
import math
import models
import tensorflow as tf
import numpy as np
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class DistillchainLstmAttentionMaxPoolingModel(models.BaseModel):
"""Max pooling over temporal weighted sums (attention) of lstm outputs."""
def create_model(self, model_input, vocab_size, num_frames,
num_mixtures=None, l2_penalty=1e-8, sub_scope="",
distillation_predictions=None,
original_input=None, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
assert distillation_predictions is not None, "distillation feature must be used"
distillchain_relu_cells = FLAGS.distillchain_relu_cells
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
num_attentions = FLAGS.lstm_attentions
batch_size, max_frames, num_features = model_input.get_shape().as_list()
mask = tf.sequence_mask(lengths=num_frames, maxlen=max_frames, dtype=tf.float32)
distill_relu = slim.fully_connected(
distillation_predictions,
distillchain_relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="distillrelu")
distill_norm = tf.nn.l2_normalize(distill_relu, dim=1)
tiled_distill_norm = tf.tile(
input=tf.expand_dims(distill_norm, axis=1),
multiples=[1,num_attentions,1])
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
attention_activations = slim.fully_connected(
tf.concat([model_input, outputs], axis=2),
num_attentions,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="attention-"+sub_scope)
# Batch x #Attentions x #Frames
attention_weights = tf.einsum("ijk,ij->ikj", tf.nn.softmax(attention_activations, dim=1), mask)
attention_weights = attention_weights / tf.reduce_sum(attention_weights, axis=2, keep_dims=True)
# Batch x #Attentions x #Features
attention_outputs = tf.einsum("ijk,ilj->ilk", outputs, attention_weights)
attention_outputs = tf.concat([attention_outputs, tiled_distill_norm], axis=2)
moe_predictions = self.sub_moe(attention_outputs, vocab_size, sub_scope="sub-moe")
predictions = tf.reshape(moe_predictions, [-1, num_attentions, vocab_size])
max_predictions = tf.reduce_max(predictions, axis=1)
return {"predictions": max_predictions}
def sub_moe(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", **unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch, [-1, vocab_size])
return final_probabilities
================================================
FILE: youtube-8m-wangheda/all_frame_models/distillchain_lstm_cnn_deep_combine_chain_model.py
================================================
import math
import models
import tensorflow as tf
import numpy as np
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class DistillchainLstmCnnDeepCombineChainModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def cnn(self,
model_input,
l2_penalty=1e-8,
num_filters = [1024, 1024, 1024],
filter_sizes = [1,2,3],
sub_scope="",
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
shift_inputs = []
for i in xrange(max(filter_sizes)):
if i == 0:
shift_inputs.append(model_input)
else:
shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:])
cnn_outputs = []
for nf, fs in zip(num_filters, filter_sizes):
sub_input = tf.concat(shift_inputs[:fs], axis=2)
sub_filter = tf.get_variable(sub_scope+"cnn-filter-len%d"%fs,
shape=[num_features*fs, nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter))
cnn_output = tf.concat(cnn_outputs, axis=2)
return cnn_output
def create_model(self, model_input, vocab_size, num_frames, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", original_input=None,
distillation_predictions=None,
**unused_params):
assert distillation_predictions is not None, "distillation feature must be used"
distillchain_relu_cells = FLAGS.distillchain_relu_cells
num_supports = FLAGS.num_supports
num_layers = FLAGS.deep_chain_layers
relu_cells = FLAGS.deep_chain_relu_cells
max_frames = model_input.get_shape().as_list()[1]
relu_layers = []
support_predictions = []
# distill
distill_relu = slim.fully_connected(
distillation_predictions,
distillchain_relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="distillrelu")
distill_norm = tf.nn.l2_normalize(distill_relu, dim=1)
relu_layers.append(distill_norm)
# mean
mask = tf.sequence_mask(num_frames, maxlen=max_frames, dtype=tf.float32)
mean_input = tf.einsum("ijk,ij->ik", model_input, mask) \
/ tf.expand_dims(tf.cast(num_frames, dtype=tf.float32), dim=1)
mean_relu = slim.fully_connected(
mean_input,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"mean-relu")
mean_relu_norm = tf.nn.l2_normalize(mean_relu, dim=1)
relu_layers.append(mean_relu_norm)
# LSTM
lstm_output = self.lstmoutput(model_input, vocab_size, num_frames)
# CNN
cnn_output = self.cnn(lstm_output, num_filters=[relu_cells,2*relu_cells,relu_cells], filter_sizes=[1,2,3], sub_scope=sub_scope+"cnn0")
max_cnn_output = tf.reduce_max(cnn_output, axis=1)
normalized_cnn_output = tf.nn.l2_normalize(max_cnn_output, dim=1)
next_input = tf.concat([normalized_cnn_output] + relu_layers, axis=1)
for layer in xrange(num_layers):
sub_prediction = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"prediction-%d"%layer)
support_predictions.append(sub_prediction)
sub_relu = slim.fully_connected(
sub_prediction,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"relu-%d"%layer)
relu_norm = tf.nn.l2_normalize(sub_relu, dim=1)
relu_layers.append(relu_norm)
cnn_output = self.cnn(lstm_output, num_filters=[relu_cells,relu_cells,relu_cells*2], filter_sizes=[1,2,3], sub_scope=sub_scope+"cnn%d"%(layer+1))
max_cnn_output = tf.reduce_max(cnn_output, axis=1)
normalized_cnn_output = tf.nn.l2_normalize(max_cnn_output, dim=1)
next_input = tf.concat([normalized_cnn_output] + relu_layers, axis=1)
main_predictions = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"-main")
support_predictions = tf.concat(support_predictions, axis=1)
return {"predictions": main_predictions, "support_predictions": support_predictions}
def sub_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", **unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
def get_mask(self, max_frames, num_frames):
mask_array = []
for i in xrange(max_frames + 1):
tmp = [0.0] * max_frames
for j in xrange(i):
tmp[j] = 1.0
mask_array.append(tmp)
mask_array = np.array(mask_array)
mask_init = tf.constant_initializer(mask_array)
mask_emb = tf.get_variable("mask_emb", shape = [max_frames + 1, max_frames],
dtype = tf.float32, trainable = False, initializer = mask_init)
mask = tf.nn.embedding_lookup(mask_emb, num_frames)
return mask
def lstmoutput(self, model_input, vocab_size, num_frames):
number_of_layers = FLAGS.lstm_layers
lstm_sizes = map(int, FLAGS.lstm_cells.split(","))
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
sub_inputs = [tf.nn.l2_normalize(x, dim=2) for x in tf.split(model_input, feature_sizes, axis = 2)]
assert len(lstm_sizes) == len(feature_sizes), \
"length of lstm_sizes (={}) != length of feature_sizes (={})".format( \
len(lstm_sizes), len(feature_sizes))
outputs = []
for i in xrange(len(feature_sizes)):
with tf.variable_scope("RNN%d" % i):
sub_input = sub_inputs[i]
lstm_size = lstm_sizes[i]
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
output, state = tf.nn.dynamic_rnn(stacked_lstm, sub_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
outputs.append(output)
# concat
final_output = tf.concat(outputs, axis=2)
return final_output
================================================
FILE: youtube-8m-wangheda/all_frame_models/distillchain_lstm_memory_deep_combine_chain_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class DistillchainLstmMemoryDeepCombineChainModel(models.BaseModel):
"""Classifier chain model of lstm memory"""
def create_model(self, model_input, vocab_size, num_frames,
l2_penalty=1e-8, sub_scope="", original_input=None,
distillation_predictions=None,
**unused_params):
assert distillation_predictions is not None, "distillation feature must be used"
distillchain_relu_cells = FLAGS.distillchain_relu_cells
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
num_supports = FLAGS.num_supports
num_layers = FLAGS.deep_chain_layers
relu_cells = FLAGS.deep_chain_relu_cells
max_frames = model_input.get_shape().as_list()[1]
relu_layers = []
support_predictions = []
# distill predictions
distill_relu = slim.fully_connected(
distillation_predictions,
distillchain_relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="distill-relu")
distill_norm = tf.nn.l2_normalize(distill_relu, dim=1)
relu_layers.append(distill_norm)
# mean input
mask = tf.sequence_mask(num_frames, maxlen=max_frames, dtype=tf.float32)
mean_input = tf.einsum("ijk,ij->ik", model_input, mask) \
/ tf.expand_dims(tf.cast(num_frames, dtype=tf.float32), dim=1)
mean_relu = slim.fully_connected(
mean_input,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"mean-relu")
mean_relu_norm = tf.nn.l2_normalize(mean_relu, dim=1)
relu_layers.append(mean_relu_norm)
lstm_output = self.sub_lstm(model_input, num_frames, lstm_size, number_of_layers, sub_scope="lstm-%d"%0)
normalized_lstm_output = tf.nn.l2_normalize(lstm_output, dim=1)
next_input = tf.concat([normalized_lstm_output] + relu_layers, axis=1)
for layer in xrange(num_layers):
sub_prediction = self.sub_moe(next_input, vocab_size, sub_scope=sub_scope+"prediction-%d"%layer)
support_predictions.append(sub_prediction)
sub_relu = slim.fully_connected(
sub_prediction,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"relu-%d"%layer)
relu_norm = tf.nn.l2_normalize(sub_relu, dim=1)
relu_layers.append(relu_norm)
lstm_output = self.sub_lstm(model_input, num_frames, lstm_size, number_of_layers, sub_scope="lstm-%d"%(layer+1))
normalized_lstm_output = tf.nn.l2_normalize(lstm_output, dim=1)
next_input = tf.concat([normalized_lstm_output] + relu_layers, axis=1)
main_predictions = self.sub_moe(next_input, vocab_size, sub_scope=sub_scope+"-main")
support_predictions = tf.concat(support_predictions, axis=1)
return {"predictions": main_predictions, "support_predictions": support_predictions}
def sub_lstm(self, model_input, num_frames, lstm_size, number_of_layers, sub_scope=""):
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
loss = 0.0
with tf.variable_scope(sub_scope+"-RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
final_state = tf.concat(map(lambda x: x.c, state), axis = 1)
return final_state
def sub_moe(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", **unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
================================================
FILE: youtube-8m-wangheda/all_frame_models/distillchain_lstm_parallel_finaloutput_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class DistillchainLstmParallelFinaloutputModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames,
distillation_predictions=None,
l2_penalty=1e-8,
**unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
assert distillation_predictions is not None, "distillation feature must be used"
relu_cells = FLAGS.distillchain_relu_cells
number_of_layers = FLAGS.lstm_layers
distill_relu = slim.fully_connected(
distillation_predictions,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="distillrelu")
distill_norm = tf.nn.l2_normalize(distill_relu, dim=1)
lstm_sizes = map(int, FLAGS.lstm_cells.split(","))
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
sub_inputs = [tf.nn.l2_normalize(x, dim=2) for x in tf.split(model_input, feature_sizes, axis = 2)]
assert len(lstm_sizes) == len(feature_sizes), \
"length of lstm_sizes (={}) != length of feature_sizes (={})".format( \
len(lstm_sizes), len(feature_sizes))
outputs = []
states = []
for i in xrange(len(feature_sizes)):
with tf.variable_scope("RNN%d" % i):
sub_input = sub_inputs[i]
lstm_size = lstm_sizes[i]
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
output, state = tf.nn.dynamic_rnn(stacked_lstm, sub_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
outputs.append(output)
states.extend(map(lambda x: x.h, state))
# concat
final_state = tf.concat(states + [distill_norm], axis = 1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=final_state,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
================================================
FILE: youtube-8m-wangheda/all_frame_models/distillchain_multiscale_cnn_lstm_model.py
================================================
import math
import models
import tensorflow as tf
import numpy as np
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class DistillchainMultiscaleCnnLstmModel(models.BaseModel):
def cnn(self,
model_input,
l2_penalty=1e-8,
num_filters = [1024, 1024, 1024],
filter_sizes = [1,2,3],
sub_scope="",
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
shift_inputs = []
for i in xrange(max(filter_sizes)):
if i == 0:
shift_inputs.append(model_input)
else:
shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:])
cnn_outputs = []
for nf, fs in zip(num_filters, filter_sizes):
sub_input = tf.concat(shift_inputs[:fs], axis=2)
sub_filter = tf.get_variable(sub_scope+"cnn-filter-len%d"%fs,
shape=[num_features*fs, nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter))
cnn_output = tf.concat(cnn_outputs, axis=2)
cnn_output = slim.batch_norm(
cnn_output,
center=True,
scale=True,
is_training=FLAGS.is_training,
scope=sub_scope+"cluster_bn")
return cnn_output
def moe(self,model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
scopename="",
**unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+scopename)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts"+scopename)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
def rnn(self, model_input, lstm_size, num_frames,
sub_scope="", **unused_params):
cell = tf.contrib.rnn.BasicLSTMCell(lstm_size, forget_bias=1.0,
state_is_tuple=True)
with tf.variable_scope("RNN-"+sub_scope):
outputs, state = tf.nn.dynamic_rnn(cell, model_input,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32)
# return final memory
return state.c
def create_model(self, model_input, vocab_size, num_frames,
distillation_predictions=None,
l2_penalty=1e-8, **unused_params):
assert distillation_predictions is not None, "distillation feature must be used"
distillchain_relu_cells = FLAGS.distillchain_relu_cells
num_layers = FLAGS.multiscale_cnn_lstm_layers
lstm_size = int(FLAGS.lstm_cells)
pool_size=2
num_filters=[256,256,512]
filter_sizes=[1,2,3]
features_size = sum(num_filters)
sub_predictions = []
cnn_input = model_input
distill_relu = slim.fully_connected(
distillation_predictions,
distillchain_relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="distillrelu")
distill_norm = tf.nn.l2_normalize(distill_relu, dim=1)
cnn_max_frames = model_input.get_shape().as_list()[1]
for layer in range(num_layers):
cnn_output = self.cnn(cnn_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn%d"%(layer+1))
cnn_output_relu = tf.nn.relu(cnn_output)
lstm_memory = self.rnn(cnn_output_relu, lstm_size, num_frames, sub_scope="rnn%d"%(layer+1))
memory_and_predictions = tf.concat([lstm_memory, distill_norm], axis=1)
sub_prediction = self.moe(memory_and_predictions, vocab_size, scopename="moe%d"%(layer+1))
sub_predictions.append(sub_prediction)
cnn_max_frames /= pool_size
max_pooled_cnn_output = tf.reduce_max(
tf.reshape(
cnn_output_relu[:, :cnn_max_frames*2, :],
[-1, cnn_max_frames, pool_size, features_size]
), axis=2)
# for the next cnn layer
cnn_input = max_pooled_cnn_output
num_frames = tf.maximum(num_frames/pool_size, 1)
support_predictions = tf.concat(sub_predictions, axis=1)
predictions = tf.add_n(sub_predictions) / len(sub_predictions)
return {"predictions": predictions,
"support_predictions": support_predictions}
================================================
FILE: youtube-8m-wangheda/all_frame_models/frame_seg_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class FrameSegModel(models.BaseModel):
def create_model(self,
model_input,
vocab_size,
num_frames,
l2_penalty=1e-8,
**unused_params):
relu_cells = FLAGS.frame_seg_relu_cells
float_num_frames = tf.cast(num_frames, tf.float32)
divisions = [0.0, 0.1, 0.5, 0.9, 1.0]
relu_layers = []
for i in xrange(4):
frame_start = float_num_frames * divisions[i]
frame_end = float_num_frames * divisions[i+1]
mean_frame = self.frame_mean(model_input, frame_start, frame_end)
mean_frame = tf.nn.l2_normalize(mean_frame, dim=1)
mean_relu = slim.fully_connected(
mean_frame,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="relu%d"%i)
relu_layers.append(tf.nn.l2_normalize(mean_relu, dim=1))
relu_layer = tf.concat(relu_layers, axis=1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=relu_layer,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
def frame_mean(self, model_input, frame_start,
frame_end, **unused_params):
max_frames = model_input.shape.as_list()[-2]
frame_start = tf.cast(frame_start, tf.int32)
frame_end = tf.cast(frame_end, tf.int32)
frame_length = tf.expand_dims(tf.cast(frame_end - frame_start, tf.float32), axis=1)
frame_mask = tf.sequence_mask(frame_end, maxlen=max_frames, dtype=tf.float32) \
- tf.sequence_mask(frame_start, maxlen=max_frames, dtype=tf.float32)
mean_frame = tf.einsum("ijk,ij->ik", model_input, frame_mask) / (0.1 + frame_length)
return mean_frame
================================================
FILE: youtube-8m-wangheda/all_frame_models/framehop_lstm_memory_deep_combine_chain_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class FramehopLstmMemoryDeepCombineChainModel(models.BaseModel):
"""Classifier chain model of lstm memory"""
def lstm(self, model_input, vocab_size, num_frames, sub_scope="",
feature_names=None, feature_sizes=None, **unused_params):
number_of_layers = FLAGS.lstm_layers
lstm_sizes = map(int, FLAGS.lstm_cells.split(","))
if feature_names is None:
feature_names = FLAGS.feature_names
if feature_sizes is None:
feature_sizes = FLAGS.feature_sizes
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
feature_names, feature_sizes)
sub_inputs = [tf.nn.l2_normalize(x, dim=2) for x in tf.split(model_input, feature_sizes, axis = 2)]
print model_input
assert len(lstm_sizes) == len(feature_sizes), \
"length of lstm_sizes (={}) != length of feature_sizes (={})".format( \
len(lstm_sizes), len(feature_sizes))
states = []
outputs = []
for i in xrange(len(feature_sizes)):
with tf.variable_scope(sub_scope+"RNN%d" % i):
print sub_scope + "RNN%d"%i, lstm_sizes
print sub_inputs
sub_input = sub_inputs[i]
lstm_size = lstm_sizes[i]
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
output, state = tf.nn.dynamic_rnn(stacked_lstm, sub_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
states.extend(map(lambda x: x.c, state))
outputs.append(output)
final_state = tf.concat(states, axis = 1)
final_output = tf.concat(outputs, axis = 2)
return final_state, final_output
def create_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", original_input=None,
dropout=False, keep_prob=None, noise_level=None,
num_frames=None,
**unused_params):
num_supports = FLAGS.num_supports
num_layers = FLAGS.deep_chain_layers
relu_cells = FLAGS.deep_chain_relu_cells
relu_type = FLAGS.deep_chain_relu_type
use_length = FLAGS.deep_chain_use_length
additional_features = []
if use_length:
print "using length as feature"
additional_features.append(self.get_length_code(num_frames))
lstm_input, lstm_frames = self.resolution(model_input, num_frames, 1)
lstm_state, lstm_output = self.lstm(lstm_input, vocab_size, num_frames=lstm_frames, sub_scope="lstm%d"%0)
next_input = tf.concat([lstm_state] + additional_features, axis=1)
support_predictions = []
for layer in xrange(num_layers):
sub_prediction = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"prediction-%d"%layer, dropout=dropout, keep_prob=keep_prob, noise_level=noise_level)
support_predictions.append(sub_prediction)
sub_activation = slim.fully_connected(
sub_prediction,
relu_cells,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"relu-%d"%layer)
if relu_type == "elu":
sub_relu = tf.nn.elu(sub_activation)
else: # default: relu
sub_relu = tf.nn.relu(sub_activation)
if noise_level is not None:
print "adding noise to sub_relu, level = ", noise_level
sub_relu = sub_relu + tf.random_normal(tf.shape(sub_relu), mean=0.0, stddev=noise_level)
relu_norm = tf.nn.l2_normalize(sub_relu, dim=1)
additional_features.append(relu_norm)
print "before resolution", lstm_output
lstm_input, lstm_frames = self.resolution(lstm_output, num_frames, 2)
print "before resolution", lstm_input
lstm_state, lstm_output = self.lstm(lstm_input, vocab_size, num_frames=lstm_frames,
feature_sizes=FLAGS.lstm_cells, sub_scope="lstm%d"%(layer+1))
next_input = tf.concat([lstm_state] + additional_features, axis=1)
main_predictions = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"-main")
support_predictions = tf.concat(support_predictions, axis=1)
return {"predictions": main_predictions, "support_predictions": support_predictions}
def sub_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="",
dropout=False, keep_prob=None, noise_level=None,
**unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
if dropout:
print "adding dropout to moe, keep_prob = ", keep_prob
model_input = tf.nn.dropout(model_input, keep_prob=keep_prob)
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
def get_length_code(self, num_frames):
code_0 = tf.cast(num_frames <= 60, dtype=tf.int32)
code_1 = tf.cast(num_frames > 60, dtype=tf.int32) * tf.cast(num_frames <= 120, dtype=tf.int32)
code_2 = tf.cast(num_frames > 120, dtype=tf.int32) * tf.cast(num_frames <= 180, dtype=tf.int32)
code_3 = tf.cast(num_frames > 180, dtype=tf.int32) * tf.cast(num_frames <= 240, dtype=tf.int32)
code_4 = tf.cast(num_frames > 240, dtype=tf.int32)
codes = map(lambda x: tf.expand_dims(x, dim=1), [code_0, code_1, code_2, code_3, code_4])
length_code = tf.cast(tf.concat(codes, axis=1), dtype=tf.float32)
return length_code
def resolution(self, model_input_raw, num_frames, resolution, method="SELECT"):
frame_dim = len(model_input_raw.get_shape()) - 2
feature_dim = len(model_input_raw.get_shape()) - 1
max_frames = model_input_raw.get_shape().as_list()[frame_dim]
num_features = model_input_raw.get_shape().as_list()[feature_dim]
if resolution > 1:
new_max_frames = max_frames / resolution
cut_frames = new_max_frames * resolution
model_input_raw = model_input_raw[:, :cut_frames, :]
model_input_raw = tf.reshape(model_input_raw, shape=[-1,new_max_frames,resolution,num_features])
if method == "MEAN":
model_input_raw = tf.reduce_mean(model_input_raw, axis=2)
elif method == "MAX":
model_input_raw = tf.reduce_max(model_input_raw, axis=2)
elif method == "SELECT":
model_input_raw = model_input_raw[:,:,resolution-1,:]
model_input = tf.nn.l2_normalize(model_input_raw, feature_dim)
num_frames = num_frames / resolution
else:
model_input = tf.nn.l2_normalize(model_input_raw, feature_dim)
return model_input, num_frames
================================================
FILE: youtube-8m-wangheda/all_frame_models/framehop_lstm_memory_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class FramehopLstmMemoryModel(models.BaseModel):
"""Classifier chain model of lstm memory"""
def lstm(self, model_input, vocab_size, num_frames, sub_scope="",
feature_names=None, feature_sizes=None, **unused_params):
number_of_layers = FLAGS.lstm_layers
lstm_sizes = map(int, FLAGS.lstm_cells.split(","))
if feature_names is None:
feature_names = FLAGS.feature_names
if feature_sizes is None:
feature_sizes = FLAGS.feature_sizes
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
feature_names, feature_sizes)
sub_inputs = [tf.nn.l2_normalize(x, dim=2) for x in tf.split(model_input, feature_sizes, axis = 2)]
print model_input
assert len(lstm_sizes) == len(feature_sizes), \
"length of lstm_sizes (={}) != length of feature_sizes (={})".format( \
len(lstm_sizes), len(feature_sizes))
states = []
outputs = []
for i in xrange(len(feature_sizes)):
with tf.variable_scope(sub_scope+"RNN%d" % i):
print sub_scope + "RNN%d"%i, lstm_sizes
print sub_inputs
sub_input = sub_inputs[i]
lstm_size = lstm_sizes[i]
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
output, state = tf.nn.dynamic_rnn(stacked_lstm, sub_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
states.extend(map(lambda x: x.c, state))
outputs.append(output)
final_state = tf.concat(states, axis = 1)
final_output = tf.concat(outputs, axis = 2)
return final_state, final_output
def create_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", original_input=None,
dropout=False, keep_prob=None, noise_level=None,
num_frames=None,
**unused_params):
num_supports = FLAGS.num_supports
num_layers = FLAGS.deep_chain_layers
relu_cells = FLAGS.deep_chain_relu_cells
relu_type = FLAGS.deep_chain_relu_type
use_length = FLAGS.deep_chain_use_length
lstm_states = []
if use_length:
print "using length as feature"
additional_features.append(self.get_length_code(num_frames))
lstm_input, lstm_frames = self.resolution(model_input, num_frames, 1)
lstm_state, lstm_output = self.lstm(lstm_input, vocab_size, num_frames=lstm_frames, sub_scope="lstm%d"%0)
lstm_states.append(lstm_state)
support_predictions = []
for layer in xrange(num_layers):
lstm_input, lstm_frames = self.resolution(lstm_output, num_frames, 2)
lstm_state, lstm_output = self.lstm(lstm_input, vocab_size, num_frames=lstm_frames,
feature_sizes=FLAGS.lstm_cells, sub_scope="lstm%d"%(layer+1))
lstm_states.append(lstm_state)
final_states = tf.concat(lstm_states, axis=1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=final_states,
original_input=model_input,
vocab_size=vocab_size,
num_frames=num_frames,
**unused_params)
def get_length_code(self, num_frames):
code_0 = tf.cast(num_frames <= 60, dtype=tf.int32)
code_1 = tf.cast(num_frames > 60, dtype=tf.int32) * tf.cast(num_frames <= 120, dtype=tf.int32)
code_2 = tf.cast(num_frames > 120, dtype=tf.int32) * tf.cast(num_frames <= 180, dtype=tf.int32)
code_3 = tf.cast(num_frames > 180, dtype=tf.int32) * tf.cast(num_frames <= 240, dtype=tf.int32)
code_4 = tf.cast(num_frames > 240, dtype=tf.int32)
codes = map(lambda x: tf.expand_dims(x, dim=1), [code_0, code_1, code_2, code_3, code_4])
length_code = tf.cast(tf.concat(codes, axis=1), dtype=tf.float32)
return length_code
def resolution(self, model_input_raw, num_frames, resolution, method="SELECT"):
frame_dim = len(model_input_raw.get_shape()) - 2
feature_dim = len(model_input_raw.get_shape()) - 1
max_frames = model_input_raw.get_shape().as_list()[frame_dim]
num_features = model_input_raw.get_shape().as_list()[feature_dim]
if resolution > 1:
new_max_frames = max_frames / resolution
cut_frames = new_max_frames * resolution
model_input_raw = model_input_raw[:, :cut_frames, :]
model_input_raw = tf.reshape(model_input_raw, shape=[-1,new_max_frames,resolution,num_features])
if method == "MEAN":
model_input_raw = tf.reduce_mean(model_input_raw, axis=2)
elif method == "MAX":
model_input_raw = tf.reduce_max(model_input_raw, axis=2)
elif method == "SELECT":
model_input_raw = model_input_raw[:,:,resolution-1,:]
model_input = tf.nn.l2_normalize(model_input_raw, feature_dim)
num_frames = num_frames / resolution
else:
model_input = tf.nn.l2_normalize(model_input_raw, feature_dim)
return model_input, num_frames
================================================
FILE: youtube-8m-wangheda/all_frame_models/gru_pooling_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class GruPoolingModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
gru_size = FLAGS.gru_cells
number_of_layers = FLAGS.gru_layers
## Batch normalize the input
stacked_gru = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.GRUCell(gru_size)
for _ in range(number_of_layers)
],
state_is_tuple=False)
loss = 0.0
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_gru, model_input,
sequence_length=num_frames,
time_major=False,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
num_frames_matrix = tf.maximum(tf.cast(tf.expand_dims(num_frames, axis=1), dtype=tf.float32), tf.ones([batch_size, 1]))
pooling_output = tf.reduce_sum(outputs, axis = 1) / num_frames_matrix
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=pooling_output,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
================================================
FILE: youtube-8m-wangheda/all_frame_models/gru_with_pooling_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class GruWithPoolingModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
gru_size = FLAGS.gru_cells
number_of_layers = FLAGS.gru_layers
## Batch normalize the input
stacked_gru = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.GRUCell(gru_size)
for _ in range(number_of_layers)
],
state_is_tuple=False)
loss = 0.0
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_gru, model_input,
sequence_length=num_frames,
time_major=False,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
num_frames_matrix = tf.maximum(tf.cast(tf.expand_dims(num_frames, axis=1), dtype=tf.float32), tf.ones([batch_size, 1]))
pooling_output = tf.reduce_sum(outputs, axis = 1) / num_frames_matrix
final_output = tf.concat([pooling_output, state], axis = 1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=final_output,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
================================================
FILE: youtube-8m-wangheda/all_frame_models/layernorm_lstm_memory_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class LayerNormLstmMemoryModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames,
dropout=False, keep_prob=None, noise_level=None,
**unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
## Batch normalize the input
if dropout:
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.LayerNormBasicLSTMCell(
lstm_size, dropout_keep_prob=keep_prob)
for _ in range(number_of_layers)
],
state_is_tuple=True)
else:
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.LayerNormBasicLSTMCell(
lstm_size)
for _ in range(number_of_layers)
],
state_is_tuple=True)
loss = 0.0
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
final_state = tf.concat(map(lambda x: x.c, state), axis = 1)
if noise_level is not None:
final_state = final_state + tf.random_normal(tf.shape(final_state), mean=0.0, stddev=noise_level)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=final_state,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
================================================
FILE: youtube-8m-wangheda/all_frame_models/logistic_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class FrameLevelLogisticModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a logistic classifier over the average of the
frame-level features.
This class is intended to be an example for implementors of frame level
models. If you want to train a model over averaged features it is more
efficient to average them beforehand rather than on the fly.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
num_frames = tf.cast(tf.expand_dims(num_frames, 1), tf.float32)
feature_size = model_input.get_shape().as_list()[2]
denominators = tf.reshape(
tf.tile(num_frames, [1, feature_size]), [-1, feature_size])
avg_pooled = tf.reduce_sum(model_input,
axis=[1]) / denominators
output = slim.fully_connected(
avg_pooled, vocab_size, activation_fn=tf.nn.sigmoid,
weights_regularizer=slim.l2_regularizer(1e-8))
return {"predictions": output}
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_advanced_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class LstmAdvancedModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.LSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=False, use_peepholes=False)
for _ in range(number_of_layers)
],
state_is_tuple=False)
loss = 0.0
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=state,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_attention_lstm_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class LstmAttentionLstmModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
l2_penalty = unused_params.get("l2_penalty", 1e-8)
max_frames = model_input.get_shape().as_list()[1]
mask_array = []
for i in xrange(max_frames + 1):
tmp = [0.0] * max_frames
for j in xrange(i):
tmp[j] = 1.0
mask_array.append(tmp)
mask_array = np.array(mask_array)
mask_init = tf.constant_initializer(mask_array)
mask_emb = tf.get_variable("mask_emb", shape = [max_frames + 1, max_frames],
dtype = tf.float32, trainable = False, initializer = mask_init)
## Batch normalize the input
stacked_cell = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
stacked_att_cell = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
loss = 0.0
with tf.variable_scope("ATT"):
att_outputs, att_state = tf.nn.dynamic_rnn(stacked_att_cell, model_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
att_final_memory = tf.concat(map(lambda x: x.c, att_state), axis = 1)
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_cell, model_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
final_memory = tf.concat(map(lambda x: x.c, state), axis = 1)
num_frames_matrix = tf.maximum(tf.cast(
tf.expand_dims(tf.expand_dims(num_frames, axis=1), axis=2),
dtype=tf.float32), 1.0)
mask = tf.expand_dims(tf.nn.embedding_lookup(mask_emb, num_frames), axis = 2)
attention_fc = slim.fully_connected(
att_outputs, 1, activation_fn=tf.nn.sigmoid,
weights_regularizer=slim.l2_regularizer(l2_penalty))
print attention_fc
attention = attention_fc * mask
attention_sum = tf.reduce_sum(attention, axis = 1, keep_dims = True) + 1e-8
attention = attention / attention_sum
print attention
attended_output = tf.reduce_sum(attention * outputs, axis = 1)
final_state = tf.concat([attended_output, att_final_memory, final_memory], axis = 1)
print "final_state", final_state
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=final_state,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_attention_max_pooling_model.py
================================================
import math
import models
import tensorflow as tf
import numpy as np
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class LstmAttentionMaxPoolingModel(models.BaseModel):
"""Max pooling over temporal weighted sums (attention) of lstm outputs."""
def create_model(self, model_input, vocab_size, num_frames,
num_mixtures=None, l2_penalty=1e-8, sub_scope="",
original_input=None, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
num_attentions = FLAGS.lstm_attentions
batch_size, max_frames, num_features = model_input.get_shape().as_list()
mask = tf.sequence_mask(lengths=num_frames, maxlen=max_frames, dtype=tf.float32)
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
attention_activations = slim.fully_connected(
tf.concat([model_input, outputs], axis=2),
num_attentions,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="attention-"+sub_scope)
# Batch x #Attentions x #Frames
attention_weights = tf.einsum("ijk,ij->ikj", tf.nn.softmax(attention_activations, dim=1), mask)
attention_weights = attention_weights / tf.reduce_sum(attention_weights, axis=2, keep_dims=True)
# Batch x #Attentions x #Features
attention_outputs = tf.einsum("ijk,ilj->ilk", outputs, attention_weights)
moe_predictions = self.sub_moe(attention_outputs, vocab_size, sub_scope="sub-moe")
predictions = tf.reshape(moe_predictions, [-1, num_attentions, vocab_size])
max_predictions = tf.reduce_max(predictions, axis=1)
return {"predictions": max_predictions}
def sub_moe(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", **unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch, [-1, vocab_size])
return final_probabilities
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_attention_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class LstmAttentionModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
l2_penalty = unused_params.get("l2_penalty", 1e-8)
max_frames = model_input.get_shape().as_list()[1]
mask_array = []
for i in xrange(max_frames + 1):
tmp = [0.0] * max_frames
for j in xrange(i):
tmp[j] = 1.0
mask_array.append(tmp)
mask_array = np.array(mask_array)
mask_init = tf.constant_initializer(mask_array)
mask_emb = tf.get_variable("mask_emb", shape = [max_frames + 1, max_frames],
dtype = tf.float32, trainable = False, initializer = mask_init)
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=False)
for _ in range(number_of_layers)
],
state_is_tuple=False)
loss = 0.0
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
num_frames_matrix = tf.maximum(tf.cast(
tf.expand_dims(tf.expand_dims(num_frames, axis=1), axis=2),
dtype=tf.float32), 1.0)
mask = tf.expand_dims(tf.nn.embedding_lookup(mask_emb, num_frames), axis = 2)
attention_fc = slim.fully_connected(
outputs, 1, activation_fn=tf.nn.sigmoid,
weights_regularizer=slim.l2_regularizer(l2_penalty))
print attention_fc
attention = attention_fc * mask
attention_sum = tf.reduce_sum(attention, axis = 1, keep_dims = True) + 1e-8
attention = attention / attention_sum
print attention
mean_input = tf.reduce_sum(attention * model_input, axis = 1)
final_state = tf.concat([mean_input, state], axis = 1)
print mean_input
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=final_state,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_auxloss_deep_combine_chain_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class LstmAuxlossDeepCombineChainModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", original_input=None, **unused_params):
"""Creates a model that use different times of output of lstm
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
"""
number_of_layers = FLAGS.lstm_layers
deep_chain_layers = FLAGS.deep_chain_layers
relu_cells = FLAGS.deep_chain_relu_cells
batch_size = tf.shape(model_input)[0]
lstm_sizes = map(int, FLAGS.lstm_cells.split(","))
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
sub_inputs = [tf.nn.l2_normalize(x, dim=2) for x in tf.split(model_input, feature_sizes, axis = 2)]
assert len(lstm_sizes) == len(feature_sizes), \
"length of lstm_sizes (={}) != length of feature_sizes (={})".format( \
len(lstm_sizes), len(feature_sizes))
outputs = []
for i in xrange(len(feature_sizes)):
with tf.variable_scope("RNN%d" % i):
sub_input = sub_inputs[i]
lstm_size = lstm_sizes[i]
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
output, state = tf.nn.dynamic_rnn(stacked_lstm, sub_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
time_major=False,
dtype=tf.float32)
outputs.append(output)
# generate time to get frame output
num_layers = deep_chain_layers + 1
timed_num_frames = []
print "num_layers =", num_layers
unit_num_frames = num_frames / num_layers
for i in xrange(deep_chain_layers):
timed_num_frames.append(unit_num_frames * (i+1))
timed_num_frames.append(num_frames)
# generate frame outputs from certain time points (num_frames - 1 to turn length into index (base 0))
timed_outputs = []
for sub_num_frames in timed_num_frames:
frame_outputs = []
for output in outputs:
frame_index = tf.stack([tf.range(batch_size), tf.maximum(sub_num_frames-1,0)], axis=1)
frame_output = tf.gather_nd(output, frame_index)
frame_outputs.append(frame_output)
timed_outputs.append(tf.concat(frame_outputs, axis=1))
# deep combine model
predictions = []
relu_layers = []
for layer in xrange(num_layers):
if relu_layers:
next_input = tf.concat([timed_outputs[layer]] + relu_layers, axis=1)
else:
next_input = timed_outputs[layer]
sub_prediction = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"prediction-%d"%layer)
predictions.append(sub_prediction)
sub_relu = slim.fully_connected(
sub_prediction,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"relu-%d"%layer)
if layer + 1 < num_layers:
relu_norm = tf.nn.l2_normalize(sub_relu, dim=1)
relu_layers.append(relu_norm)
main_predictions = predictions[-1]
support_predictions = tf.concat(predictions[:-1], axis=1)
return {"predictions": main_predictions, "support_predictions": support_predictions}
def sub_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", **unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_cnn_deep_combine_chain_model.py
================================================
import math
import models
import tensorflow as tf
import numpy as np
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class LstmCnnDeepCombineChainModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def cnn(self,
model_input,
l2_penalty=1e-8,
num_filters = [1024, 1024, 1024],
filter_sizes = [1,2,3],
sub_scope="",
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
shift_inputs = []
for i in xrange(max(filter_sizes)):
if i == 0:
shift_inputs.append(model_input)
else:
shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:])
cnn_outputs = []
for nf, fs in zip(num_filters, filter_sizes):
sub_input = tf.concat(shift_inputs[:fs], axis=2)
sub_filter = tf.get_variable(sub_scope+"cnn-filter-len%d"%fs,
shape=[num_features*fs, nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter))
cnn_output = tf.concat(cnn_outputs, axis=2)
return cnn_output
def create_model(self, model_input, vocab_size, num_frames, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", original_input=None, **unused_params):
num_supports = FLAGS.num_supports
num_layers = FLAGS.deep_chain_layers
relu_cells = FLAGS.deep_chain_relu_cells
max_frames = model_input.get_shape().as_list()[1]
relu_layers = []
support_predictions = []
mask = self.get_mask(max_frames, num_frames)
mean_input = tf.einsum("ijk,ij->ik", model_input, mask) \
/ tf.expand_dims(tf.cast(num_frames, dtype=tf.float32), dim=1)
# LSTM
lstm_output = self.lstmoutput(model_input, vocab_size, num_frames)
mean_relu = slim.fully_connected(
mean_input,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"mean-relu")
mean_relu_norm = tf.nn.l2_normalize(mean_relu, dim=1)
relu_layers.append(mean_relu_norm)
cnn_output = self.cnn(lstm_output, num_filters=[relu_cells,2*relu_cells,relu_cells], filter_sizes=[1,2,3], sub_scope=sub_scope+"cnn0")
max_cnn_output = tf.reduce_max(cnn_output, axis=1)
normalized_cnn_output = tf.nn.l2_normalize(max_cnn_output, dim=1)
next_input = normalized_cnn_output
for layer in xrange(num_layers):
sub_prediction = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"prediction-%d"%layer)
support_predictions.append(sub_prediction)
sub_relu = slim.fully_connected(
sub_prediction,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"relu-%d"%layer)
relu_norm = tf.nn.l2_normalize(sub_relu, dim=1)
relu_layers.append(relu_norm)
cnn_output = self.cnn(lstm_output, num_filters=[relu_cells,relu_cells,relu_cells*2], filter_sizes=[1,2,3], sub_scope=sub_scope+"cnn%d"%(layer+1))
max_cnn_output = tf.reduce_max(cnn_output, axis=1)
normalized_cnn_output = tf.nn.l2_normalize(max_cnn_output, dim=1)
next_input = tf.concat([normalized_cnn_output] + relu_layers, axis=1)
main_predictions = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"-main")
support_predictions = tf.concat(support_predictions, axis=1)
return {"predictions": main_predictions, "support_predictions": support_predictions}
def sub_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", **unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
def get_mask(self, max_frames, num_frames):
mask_array = []
for i in xrange(max_frames + 1):
tmp = [0.0] * max_frames
for j in xrange(i):
tmp[j] = 1.0
mask_array.append(tmp)
mask_array = np.array(mask_array)
mask_init = tf.constant_initializer(mask_array)
mask_emb = tf.get_variable("mask_emb", shape = [max_frames + 1, max_frames],
dtype = tf.float32, trainable = False, initializer = mask_init)
mask = tf.nn.embedding_lookup(mask_emb, num_frames)
return mask
def lstmoutput(self, model_input, vocab_size, num_frames):
number_of_layers = FLAGS.lstm_layers
lstm_sizes = map(int, FLAGS.lstm_cells.split(","))
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
sub_inputs = [tf.nn.l2_normalize(x, dim=2) for x in tf.split(model_input, feature_sizes, axis = 2)]
assert len(lstm_sizes) == len(feature_sizes), \
"length of lstm_sizes (={}) != length of feature_sizes (={})".format( \
len(lstm_sizes), len(feature_sizes))
outputs = []
for i in xrange(len(feature_sizes)):
with tf.variable_scope("RNN%d" % i):
sub_input = sub_inputs[i]
lstm_size = lstm_sizes[i]
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
output, state = tf.nn.dynamic_rnn(stacked_lstm, sub_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
outputs.append(output)
# concat
final_output = tf.concat(outputs, axis=2)
return final_output
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_divided_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class LstmDividedModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
with tf.device("/gpu:1"):
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=False)
for _ in range(number_of_layers)
],
state_is_tuple=False)
loss = 0.0
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
with tf.device("/gpu:0"):
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
predictions = aggregated_model().create_model(
model_input=state,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
return predictions
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_look_back_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class LstmLookBackModel(models.BaseModel):
def shift(self,
model_input,
shift_width,
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
shift_inputs = []
for i in xrange(shift_width):
if i == 0:
shift_inputs.append(model_input)
else:
shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:])
shift_output = tf.concat(shift_inputs, axis=2)
return shift_output
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
number_of_layers = FLAGS.lstm_layers
shift_width = FLAGS.lstm_look_back
lstm_sizes = map(int, FLAGS.lstm_cells.split(","))
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
sub_inputs = [tf.nn.l2_normalize(x, dim=2) for x in tf.split(model_input, feature_sizes, axis = 2)]
assert len(lstm_sizes) == len(feature_sizes), \
"length of lstm_sizes (={}) != length of feature_sizes (={})".format( \
len(lstm_sizes), len(feature_sizes))
if shift_width < 1:
shift_width = 1
outputs = []
states = []
for i in xrange(len(feature_sizes)):
with tf.variable_scope("RNN%d" % i):
sub_input = sub_inputs[i]
lstm_size = lstm_sizes[i]
look_back = self.shift(sub_input, shift_width=shift_width)
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
output, state = tf.nn.dynamic_rnn(stacked_lstm, look_back,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
outputs.append(output)
states.extend(map(lambda x: x.h, state))
# concat
final_state = tf.concat(states, axis = 1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=final_state,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_memory_chain_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class LstmMemoryChainModel(models.BaseModel):
"""Classifier chain model of lstm memory"""
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
num_supports = FLAGS.num_supports
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
loss = 0.0
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
final_state = tf.concat(map(lambda x: x.c, state), axis = 1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
support_predictions = aggregated_model().create_model(
model_input=final_state,
original_input=model_input,
vocab_size=num_supports,
sub_scope="support",
**unused_params)
support_predictions = support_predictions["predictions"]
main_state = tf.concat([final_state, support_predictions], axis=1)
predictions = aggregated_model().create_model(
model_input=main_state,
original_input=model_input,
vocab_size=vocab_size,
sub_scope="main",
**unused_params)
predictions["support_predictions"] = support_predictions
return predictions
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_memory_deep_chain_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class LstmMemoryDeepChainModel(models.BaseModel):
"""Classifier chain model of lstm memory"""
def create_model(self, model_input, vocab_size, num_frames,
l2_penalty=1e-8, sub_scope="", original_input=None, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
num_supports = FLAGS.num_supports
num_layers = FLAGS.deep_chain_layers
relu_cells = FLAGS.deep_chain_relu_cells
lstm_output = self.sub_lstm(model_input, num_frames, lstm_size, number_of_layers, sub_scope="lstm-%d"%0)
next_input = lstm_output
support_predictions = []
for layer in xrange(num_layers):
sub_prediction = self.sub_moe(next_input, vocab_size, sub_scope=sub_scope+"prediction-%d"%layer)
sub_relu = slim.fully_connected(
sub_prediction,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"relu-%d"%layer)
relu_norm = tf.nn.l2_normalize(sub_relu, dim=1)
lstm_output = self.sub_lstm(model_input, num_frames, lstm_size, number_of_layers, sub_scope="lstm-%d"%(layer+1))
next_input = tf.concat([lstm_output, relu_norm], axis=1)
support_predictions.append(sub_prediction)
main_predictions = self.sub_moe(next_input, vocab_size, sub_scope=sub_scope+"-main")
support_predictions = tf.concat(support_predictions, axis=1)
return {"predictions": main_predictions, "support_predictions": support_predictions}
def sub_lstm(self, model_input, num_frames, lstm_size, number_of_layers, sub_scope=""):
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
loss = 0.0
with tf.variable_scope(sub_scope+"-RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
final_state = tf.concat(map(lambda x: x.c, state), axis = 1)
return final_state
def sub_moe(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", **unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_memory_input_chain_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class LstmMemoryInputChainModel(models.BaseModel):
"""Classifier chain model of lstm memory"""
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
num_supports = FLAGS.num_supports
num_support_embedding = 200
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
mean_input = tf.reduce_mean(model_input, axis=1)
support_predictions = aggregated_model().create_model(
model_input=mean_input,
original_input=model_input,
vocab_size=num_supports,
sub_scope="support",
**unused_params)
support_predictions = support_predictions["predictions"]
support_activations = tf.stop_gradient(support_predictions * 0.06 - 0.03)
max_frames = model_input.shape.as_list()[1]
support_input = tf.tile(
tf.expand_dims(support_activations, axis=1),
multiples=[1, max_frames, 1])
actual_input = tf.concat([model_input, support_input], axis=2)
normalized_input = tf.nn.l2_normalize(actual_input, dim=2)
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
loss = 0.0
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, normalized_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
final_state = tf.concat(map(lambda x: x.c, state), axis = 1)
predictions = aggregated_model().create_model(
model_input=final_state,
original_input=model_input,
vocab_size=vocab_size,
sub_scope="main",
**unused_params)
predictions["support_predictions"] = support_predictions
return predictions
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_memory_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class LstmMemoryModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames,
dropout=False, keep_prob=None, noise_level=None,
**unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
## Batch normalize the input
if dropout:
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.DropoutWrapper(
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True),
input_keep_prob=keep_prob)
for _ in range(number_of_layers)
],
state_is_tuple=True)
else:
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
loss = 0.0
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
final_state = tf.concat(map(lambda x: x.c, state), axis = 1)
if noise_level is not None:
final_state = final_state + tf.random_normal(tf.shape(final_state), mean=0.0, stddev=noise_level)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=final_state,
original_input=model_input,
vocab_size=vocab_size,
num_frames=num_frames,
**unused_params)
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_memory_multitask_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class LstmMemoryMultitaskModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
num_supports = FLAGS.num_supports
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
loss = 0.0
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
final_state = tf.concat(map(lambda x: x.c, state), axis = 1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
predictions = aggregated_model().create_model(
model_input=final_state,
original_input=model_input,
vocab_size=vocab_size,
sub_scope="main",
**unused_params)
support_model = getattr(video_level_models,
FLAGS.video_level_classifier_support_model)
support_predictions = support_model().create_model(
model_input=final_state,
original_input=model_input,
vocab_size=num_supports,
sub_scope="support",
**unused_params)
predictions["support_predictions"] = support_predictions["predictions"]
return predictions
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_memory_normalization_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class LstmMemoryNormalizationModel(models.BaseModel):
"""layer normalization (not the Hinton one)"""
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
normalize_class = getattr(self, FLAGS.lstm_normalization, self.identical)
outputs, state = model_input, []
for layer in xrange(number_of_layers):
with tf.variable_scope("RNN-layer%d" % layer):
outputs = normalize_class(outputs)
cell = tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
layer_outputs, layer_state = tf.nn.dynamic_rnn(cell, outputs,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
state.append(layer_state)
outputs = layer_outputs
final_state = tf.concat(map(lambda x: x.c, state), axis = 1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=final_state,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
def layer_normalize(self, input_raw, epsilon=1e-8):
feature_dim = len(input_raw.get_shape()) - 1
mean_input = tf.reduce_mean(input_raw, axis=feature_dim, keep_dims=True)
std_input = tf.sqrt(tf.reduce_mean(tf.square(input_raw-mean_input), axis=feature_dim, keep_dims=True))
std_input = tf.maximum(std_input, epsilon)
output = (input_raw - mean_input) / std_input
return output
def l2_normalize(self, input_raw, epsilon=1e-8):
feature_dim = len(input_raw.get_shape()) - 1
output = tf.nn.l2_normalize(input_raw, dim=feature_dim)
return output
def identical(self, input_raw, epsilon=1e-8):
return input_raw
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_memory_parallel_chain_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class LstmMemoryParallelChainModel(models.BaseModel):
"""Classifier chain model of lstm memory"""
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
support_lstm_size = lstm_size / 2
number_of_layers = FLAGS.lstm_layers
num_supports = FLAGS.num_supports
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
stacked_lstm_support = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
support_lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
with tf.variable_scope("RNN-support"):
support_outputs, support_state = tf.nn.dynamic_rnn(stacked_lstm_support, model_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
support_final_state = tf.concat(map(lambda x: x.c, support_state), axis = 1)
support_predictions = aggregated_model().create_model(
model_input=support_final_state,
original_input=model_input,
vocab_size=num_supports,
sub_scope="support",
**unused_params)
support_predictions = support_predictions["predictions"]
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
with tf.variable_scope("RNN-main"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
final_state = tf.concat(map(lambda x: x.c, state), axis = 1)
main_state = tf.concat([final_state, support_predictions], axis=1)
predictions = aggregated_model().create_model(
model_input=main_state,
original_input=model_input,
vocab_size=vocab_size,
sub_scope="main",
**unused_params)
predictions["support_predictions"] = support_predictions
return predictions
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class LstmModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=False)
for _ in range(number_of_layers)
],
state_is_tuple=False)
loss = 0.0
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
predictions = aggregated_model().create_model(
model_input=state,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
print predictions
return predictions
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_multi_attention_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class LstmMultiAttentionModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
attention_size = FLAGS.attention_size
l2_penalty = unused_params.get("l2_penalty", 1e-8)
max_frames = model_input.get_shape().as_list()[1]
mask_array = []
for i in xrange(max_frames + 1):
tmp = [0.0] * max_frames
for j in xrange(i):
tmp[j] = 1.0
mask_array.append(tmp)
mask_array = np.array(mask_array)
mask_init = tf.constant_initializer(mask_array)
mask_emb = tf.get_variable("mask_emb", shape = [max_frames + 1, max_frames],
dtype = tf.float32, trainable = False, initializer = mask_init)
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
loss = 0.0
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
num_frames_matrix = tf.maximum(tf.cast(
tf.expand_dims(tf.expand_dims(num_frames, axis=1), axis=2),
dtype=tf.float32), 1.0)
mask = tf.expand_dims(tf.nn.embedding_lookup(mask_emb, num_frames), axis = 2)
attention_fc = slim.fully_connected(
outputs, attention_size, activation_fn=tf.nn.sigmoid,
weights_regularizer=slim.l2_regularizer(l2_penalty))
print attention_fc
attention = attention_fc * mask
attention_sum = tf.reduce_sum(attention, axis = 1, keep_dims = True) + 1e-8
attention = attention / attention_sum
print attention, model_input
attention_input = tf.einsum("ijk,ijl->ikl", attention, model_input)
print attention_input
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
attention_output = aggregated_model().create_model(
model_input=attention_input,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)["predictions"]
print attention_output
final_output = tf.reduce_max(tf.reshape(attention_output, [-1, attention_size, vocab_size]), axis = 1)
return {"predictions": final_output}
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_multi_pooling_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class LstmMultiPoolingModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
batch_size = FLAGS.batch_size
pooled_output = []
outputs = model_input
num_frames_matrix = tf.maximum(tf.cast(tf.expand_dims(num_frames, axis=1), dtype=tf.float32), tf.ones([batch_size, 1]))
pooled_output.append(tf.reduce_sum(outputs, axis = 1) / num_frames_matrix)
for layer in xrange(number_of_layers):
## Batch normalize the input
cell = tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=False)
loss = 0.0
with tf.variable_scope("RNN-layer%d" % layer):
outputs, state = tf.nn.dynamic_rnn(cell, outputs,
sequence_length=num_frames,
time_major=False,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
pooled_output.append(tf.reduce_sum(outputs, axis = 1) / num_frames_matrix)
final_output = tf.concat(pooled_output, axis = 1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=final_output,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_parallel_finaloutput_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class LstmParallelFinaloutputModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
number_of_layers = FLAGS.lstm_layers
lstm_sizes = map(int, FLAGS.lstm_cells.split(","))
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
sub_inputs = [tf.nn.l2_normalize(x, dim=2) for x in tf.split(model_input, feature_sizes, axis = 2)]
assert len(lstm_sizes) == len(feature_sizes), \
"length of lstm_sizes (={}) != length of feature_sizes (={})".format( \
len(lstm_sizes), len(feature_sizes))
outputs = []
states = []
for i in xrange(len(feature_sizes)):
with tf.variable_scope("RNN%d" % i):
sub_input = sub_inputs[i]
lstm_size = lstm_sizes[i]
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
output, state = tf.nn.dynamic_rnn(stacked_lstm, sub_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
outputs.append(output)
states.extend(map(lambda x: x.h, state))
# concat
final_state = tf.concat(states, axis = 1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=final_state,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_parallel_memory_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class LstmParallelMemoryModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
number_of_layers = FLAGS.lstm_layers
lstm_sizes = map(int, FLAGS.lstm_cells.split(","))
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
sub_inputs = [tf.nn.l2_normalize(x, dim=2) for x in tf.split(model_input, feature_sizes, axis = 2)]
assert len(lstm_sizes) == len(feature_sizes), \
"length of lstm_sizes (={}) != length of feature_sizes (={})".format( \
len(lstm_sizes), len(feature_sizes))
loss = 0.0
outputs = []
states = []
for i in xrange(len(feature_sizes)):
with tf.variable_scope("RNN%d" % i):
sub_input = sub_inputs[i]
lstm_size = lstm_sizes[i]
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
output, state = tf.nn.dynamic_rnn(stacked_lstm, sub_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
outputs.append(output)
states.extend(map(lambda x: x.c, state))
# concat
final_state = tf.concat(states, axis = 1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=final_state,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_parallel_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class LstmParallelModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
number_of_layers = FLAGS.lstm_layers
lstm_sizes = map(int, FLAGS.lstm_cells.split(","))
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
sub_inputs = tf.split(model_input, feature_sizes, axis = 2)
assert len(lstm_sizes) == len(feature_sizes), \
"length of lstm_sizes (={}) != length of feature_sizes (={})".format( \
len(lstm_sizes), len(feature_sizes))
loss = 0.0
outputs = []
states = []
for i in xrange(len(feature_sizes)):
with tf.variable_scope("RNN%d" % i):
sub_input = sub_inputs[i]
lstm_size = lstm_sizes[i]
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=False)
for _ in range(number_of_layers)
],
state_is_tuple=False)
output, state = tf.nn.dynamic_rnn(stacked_lstm, sub_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
outputs.append(output)
states.append(state)
# concat
final_state = tf.concat(states, axis = 1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=final_state,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_pooling_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class LstmPoolingModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
batch_size = FLAGS.batch_size
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=False)
for _ in range(number_of_layers)
],
state_is_tuple=False)
loss = 0.0
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
time_major=False,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
num_frames_matrix = tf.maximum(tf.cast(tf.expand_dims(num_frames, axis=1), dtype=tf.float32), tf.ones([batch_size, 1]))
pooling_output = tf.reduce_sum(outputs, axis = 1) / num_frames_matrix
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=pooling_output,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_positional_attention_max_pooling_model.py
================================================
import math
import models
import tensorflow as tf
import numpy as np
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class LstmPositionalAttentionMaxPoolingModel(models.BaseModel):
"""Max pooling over temporal weighted sums (attention) of lstm outputs."""
def create_model(self, model_input, vocab_size, num_frames,
num_mixtures=None, l2_penalty=1e-8, sub_scope="",
original_input=None, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
num_attentions = FLAGS.lstm_attentions
batch_size, max_frames, num_features = model_input.get_shape().as_list()
mask = tf.sequence_mask(lengths=num_frames, maxlen=max_frames, dtype=tf.float32)
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
positional_embedding = self.get_positional_embedding(model_input, num_frames)
mean_input = self.get_mean_input(model_input, num_frames)
attention_activations = slim.fully_connected(
tf.concat([model_input, positional_embedding, mean_input, outputs], axis=2),
num_attentions,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="attention-"+sub_scope)
# Batch x #Attentions x #Frames
attention_weights = tf.einsum("ijk,ij->ikj", tf.nn.softmax(attention_activations, dim=1), mask)
attention_weights = attention_weights / tf.reduce_sum(attention_weights, axis=2, keep_dims=True)
# Batch x #Attentions x #Features
attention_outputs = tf.einsum("ijk,ilj->ilk", outputs, attention_weights)
moe_predictions = self.sub_moe(attention_outputs, vocab_size, sub_scope="sub-moe")
predictions = tf.reshape(moe_predictions, [-1, num_attentions, vocab_size])
max_predictions = tf.reduce_max(predictions, axis=1)
return {"predictions": max_predictions}
def get_positional_embedding(self, model_input, num_frames, l2_penalty=1e-8):
embedding_size = FLAGS.positional_embedding_size
batch_size, max_frames, num_features = model_input.get_shape().as_list()
positional_embedding = tf.get_variable("positional_embedding", dtype=tf.float32,
shape=[1, max_frames, embedding_size],
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
tiled_embedding = tf.tile(positional_embedding, multiples=[tf.shape(model_input)[0],1,1])
return tiled_embedding
def get_mean_input(self, model_input, num_frames):
batch_size, max_frames, num_features = model_input.get_shape().as_list()
mask = tf.sequence_mask(lengths=num_frames, maxlen=max_frames, dtype=tf.float32)
mean_input = tf.einsum("ijk,ij->ik", model_input, mask) / tf.expand_dims(tf.cast(num_frames, dtype=tf.float32), dim=1)
tiled_mean_input = tf.tile(tf.expand_dims(mean_input, dim=1), multiples=[1,max_frames,1])
return tiled_mean_input
def sub_moe(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", **unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch, [-1, vocab_size])
return final_probabilities
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_with_mean_input_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class LstmWithMeanInputModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
batch_size = FLAGS.batch_size
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=False)
for _ in range(number_of_layers)
],
state_is_tuple=False)
loss = 0.0
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
time_major=False,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
with tf.device("/gpu:0"):
num_frames_matrix = tf.maximum(tf.cast(tf.expand_dims(num_frames, axis=1), dtype=tf.float32), 1.0)
mean_input = tf.reduce_sum(model_input, axis = 1) / num_frames_matrix
final_output = tf.concat([mean_input, state], axis = 1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
predictions = aggregated_model().create_model(
model_input=final_output,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
return predictions
================================================
FILE: youtube-8m-wangheda/all_frame_models/lstm_with_pooling_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class LstmWithPoolingModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=False)
for _ in range(number_of_layers)
],
state_is_tuple=False)
loss = 0.0
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
time_major=False,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
num_frames_matrix = tf.maximum(tf.cast(tf.expand_dims(num_frames, axis=1), dtype=tf.float32), 1.0)
pooling_output = tf.reduce_sum(outputs, axis = 1) / num_frames_matrix
final_output = tf.concat([pooling_output, state], axis = 1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=final_output,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
================================================
FILE: youtube-8m-wangheda/all_frame_models/mm_lstm_memory_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class MatchingMatrixLstmMemoryModel(models.BaseModel):
def matching_matrix(self,
model_input,
vocab_size,
l2_penalty=1e-8,
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
embedding_size = FLAGS.mm_label_embedding
model_input = tf.reshape(model_input, [-1, num_features])
frame_relu = slim.fully_connected(
model_input,
embedding_size,
activation_fn=tf.nn.relu,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="mm_relu")
frame_activation = slim.fully_connected(
frame_relu,
embedding_size,
activation_fn=tf.nn.tanh,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="mm_activation")
label_embedding = tf.get_variable("label_embedding", shape=[vocab_size,embedding_size],
dtype=tf.float32, initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.5),
regularizer=slim.l2_regularizer(l2_penalty), trainable=True)
mm_matrix = tf.einsum("ik,jk->ij", frame_activation, label_embedding)
mm_output = tf.reshape(mm_matrix, [-1,max_frames,vocab_size])
return mm_output
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
mm_output = self.matching_matrix(model_input, vocab_size)
normalized_mm_output = tf.nn.l2_normalize(mm_output, dim=2)
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
loss = 0.0
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, normalized_mm_output,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
final_state = tf.concat(map(lambda x: x.c, state), axis = 1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=final_state,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
================================================
FILE: youtube-8m-wangheda/all_frame_models/multi_view_cnn_deep_combine_chain_model.py
================================================
import math
import models
import tensorflow as tf
import numpy as np
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class MultiViewCnnDeepCombineChainModel(models.BaseModel):
def cnn(self,
model_input,
l2_penalty=1e-8,
num_filters = [1024, 1024, 1024],
filter_sizes = [1,2,3],
sub_scope="",
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
shift_inputs = []
for i in xrange(max(filter_sizes)):
if i == 0:
shift_inputs.append(model_input)
else:
shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:])
cnn_outputs = []
for nf, fs in zip(num_filters, filter_sizes):
sub_input = tf.concat(shift_inputs[:fs], axis=2)
sub_filter = tf.get_variable(sub_scope+"cnn-filter-len%d"%fs,
shape=[num_features*fs, nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter))
cnn_output = tf.concat(cnn_outputs, axis=2)
return cnn_output
def create_model(self, model_input, vocab_size, num_frames, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", original_input=None, **unused_params):
num_supports = FLAGS.num_supports
num_layers = FLAGS.deep_chain_layers
relu_cells = FLAGS.deep_chain_relu_cells
max_frames = model_input.get_shape().as_list()[1]
relu_layers = []
support_predictions = []
mask = self.get_mask(max_frames, num_frames)
mean_input = tf.einsum("ijk,ij->ik", model_input, mask) \
/ tf.expand_dims(tf.cast(num_frames, dtype=tf.float32), dim=1)
mean_relu = slim.fully_connected(
mean_input,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"mean-relu")
mean_relu_norm = tf.nn.l2_normalize(mean_relu, dim=1)
relu_layers.append(mean_relu_norm)
cnn_output = self.cnn(model_input, num_filters=[relu_cells,2*relu_cells,relu_cells], filter_sizes=[1,2,3], sub_scope=sub_scope+"cnn0")
mv_cnn_output = self.multiview(cnn_output, axis=1)
normalized_cnn_output = tf.nn.l2_normalize(mv_cnn_output, dim=1)
next_input = normalized_cnn_output
for layer in xrange(num_layers):
sub_prediction = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"prediction-%d"%layer)
support_predictions.append(sub_prediction)
sub_relu = slim.fully_connected(
sub_prediction,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"relu-%d"%layer)
relu_norm = tf.nn.l2_normalize(sub_relu, dim=1)
relu_layers.append(relu_norm)
cnn_output = self.cnn(model_input, num_filters=[relu_cells,relu_cells,relu_cells*2], filter_sizes=[1,2,3], sub_scope=sub_scope+"cnn%d"%(layer+1))
mv_cnn_output = self.multiview(cnn_output, axis=1)
normalized_cnn_output = tf.nn.l2_normalize(mv_cnn_output, dim=1)
next_input = tf.concat([mean_input, normalized_cnn_output] + relu_layers, axis=1)
main_predictions = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"-main")
support_predictions = tf.concat(support_predictions, axis=1)
return {"predictions": main_predictions, "support_predictions": support_predictions}
def multiview(self, cnn_output, axis=1):
max_view = tf.reduce_max(cnn_output, axis=1)
mean_view = tf.reduce_mean(cnn_output, axis=1)
min_view = tf.reduce_min(cnn_output, axis=1)
multi_view = tf.concat([max_view, mean_view, min_view], axis=1)
return multi_view
def sub_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", **unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
def get_mask(self, max_frames, num_frames):
mask_array = []
for i in xrange(max_frames + 1):
tmp = [0.0] * max_frames
for j in xrange(i):
tmp[j] = 1.0
mask_array.append(tmp)
mask_array = np.array(mask_array)
mask_init = tf.constant_initializer(mask_array)
mask_emb = tf.get_variable("mask_emb", shape = [max_frames + 1, max_frames],
dtype = tf.float32, trainable = False, initializer = mask_init)
mask = tf.nn.embedding_lookup(mask_emb, num_frames)
return mask
================================================
FILE: youtube-8m-wangheda/all_frame_models/multires_lstm_memory_deep_combine_chain_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class MultiresLstmMemoryDeepCombineChainModel(models.BaseModel):
"""Classifier chain model of lstm memory"""
def lstm(self, model_input, vocab_size, num_frames, sub_scope="", **unused_params):
number_of_layers = FLAGS.lstm_layers
lstm_sizes = map(int, FLAGS.lstm_cells.split(","))
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
sub_inputs = [tf.nn.l2_normalize(x, dim=2) for x in tf.split(model_input, feature_sizes, axis = 2)]
assert len(lstm_sizes) == len(feature_sizes), \
"length of lstm_sizes (={}) != length of feature_sizes (={})".format( \
len(lstm_sizes), len(feature_sizes))
states = []
for i in xrange(len(feature_sizes)):
with tf.variable_scope(sub_scope+"RNN%d" % i):
sub_input = sub_inputs[i]
lstm_size = lstm_sizes[i]
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
output, state = tf.nn.dynamic_rnn(stacked_lstm, sub_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
states.extend(map(lambda x: x.c, state))
final_state = tf.concat(states, axis = 1)
return final_state
def create_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", original_input=None,
dropout=False, keep_prob=None, noise_level=None,
num_frames=None,
**unused_params):
num_supports = FLAGS.num_supports
num_layers = FLAGS.deep_chain_layers
relu_cells = FLAGS.deep_chain_relu_cells
relu_type = FLAGS.deep_chain_relu_type
use_length = FLAGS.deep_chain_use_length
num_resolutions = [int(2**(num_layers-i)) for i in xrange(num_layers+1)]
additional_features = []
if use_length:
print "using length as feature"
additional_features.append(self.get_length_code(num_frames))
lstm_input, lstm_frames = self.resolution(model_input, num_frames, num_resolutions[0])
lstm_output = self.lstm(lstm_input, vocab_size, num_frames=lstm_frames, sub_scope="lstm%d"%0)
next_input = tf.concat([lstm_output] + additional_features, axis=1)
support_predictions = []
for layer in xrange(num_layers):
sub_prediction = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"prediction-%d"%layer, dropout=dropout, keep_prob=keep_prob, noise_level=noise_level)
support_predictions.append(sub_prediction)
sub_activation = slim.fully_connected(
sub_prediction,
relu_cells,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"relu-%d"%layer)
if relu_type == "elu":
sub_relu = tf.nn.elu(sub_activation)
else: # default: relu
sub_relu = tf.nn.relu(sub_activation)
if noise_level is not None:
print "adding noise to sub_relu, level = ", noise_level
sub_relu = sub_relu + tf.random_normal(tf.shape(sub_relu), mean=0.0, stddev=noise_level)
relu_norm = tf.nn.l2_normalize(sub_relu, dim=1)
additional_features.append(relu_norm)
lstm_input, lstm_frames = self.resolution(model_input, num_frames, num_resolutions[layer+1])
lstm_output = self.lstm(lstm_input, vocab_size, num_frames=lstm_frames, sub_scope="lstm%d"%(layer+1))
next_input = tf.concat([lstm_output] + additional_features, axis=1)
main_predictions = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"-main")
support_predictions = tf.concat(support_predictions, axis=1)
return {"predictions": main_predictions, "support_predictions": support_predictions}
def sub_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="",
dropout=False, keep_prob=None, noise_level=None,
**unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
if dropout:
print "adding dropout to moe, keep_prob = ", keep_prob
model_input = tf.nn.dropout(model_input, keep_prob=keep_prob)
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
def get_length_code(self, num_frames):
code_0 = tf.cast(num_frames <= 60, dtype=tf.int32)
code_1 = tf.cast(num_frames > 60, dtype=tf.int32) * tf.cast(num_frames <= 120, dtype=tf.int32)
code_2 = tf.cast(num_frames > 120, dtype=tf.int32) * tf.cast(num_frames <= 180, dtype=tf.int32)
code_3 = tf.cast(num_frames > 180, dtype=tf.int32) * tf.cast(num_frames <= 240, dtype=tf.int32)
code_4 = tf.cast(num_frames > 240, dtype=tf.int32)
codes = map(lambda x: tf.expand_dims(x, dim=1), [code_0, code_1, code_2, code_3, code_4])
length_code = tf.cast(tf.concat(codes, axis=1), dtype=tf.float32)
return length_code
def resolution(self, model_input_raw, num_frames, resolution):
frame_dim = len(model_input_raw.get_shape()) - 2
feature_dim = len(model_input_raw.get_shape()) - 1
max_frames = model_input_raw.get_shape().as_list()[frame_dim]
num_features = model_input_raw.get_shape().as_list()[feature_dim]
if resolution > 1:
new_max_frames = max_frames / resolution
cut_frames = new_max_frames * resolution
model_input_raw = model_input_raw[:, :cut_frames, :]
model_input_raw = tf.reshape(model_input_raw, shape=[-1,new_max_frames,resolution,num_features])
model_input_raw = tf.reduce_mean(model_input_raw, axis=2)
model_input = tf.nn.l2_normalize(model_input_raw, feature_dim)
num_frames = num_frames / resolution
else:
model_input = tf.nn.l2_normalize(model_input_raw, feature_dim)
return model_input, num_frames
================================================
FILE: youtube-8m-wangheda/all_frame_models/multiscale_cnn_lstm_model.py
================================================
import math
import models
import tensorflow as tf
import numpy as np
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class MultiscaleCnnLstmModel(models.BaseModel):
def cnn(self,
model_input,
l2_penalty=1e-8,
num_filters = [1024, 1024, 1024],
filter_sizes = [1,2,3],
sub_scope="",
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
shift_inputs = []
for i in xrange(max(filter_sizes)):
if i == 0:
shift_inputs.append(model_input)
else:
shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:])
cnn_outputs = []
for nf, fs in zip(num_filters, filter_sizes):
sub_input = tf.concat(shift_inputs[:fs], axis=2)
sub_filter = tf.get_variable(sub_scope+"cnn-filter-len%d"%fs,
shape=[num_features*fs, nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter))
cnn_output = tf.concat(cnn_outputs, axis=2)
cnn_output = slim.batch_norm(
cnn_output,
center=True,
scale=True,
is_training=FLAGS.is_training,
scope=sub_scope+"cluster_bn")
return cnn_output
def moe(self,model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
scopename="",
**unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+scopename)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts"+scopename)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
def rnn(self, model_input, lstm_size, num_frames,
sub_scope="", **unused_params):
cell = tf.contrib.rnn.BasicLSTMCell(lstm_size, forget_bias=1.0,
state_is_tuple=True)
with tf.variable_scope("RNN-"+sub_scope):
outputs, state = tf.nn.dynamic_rnn(cell, model_input,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32)
# return final memory
return state.c
def create_model(self, model_input, vocab_size, num_frames,
l2_penalty=1e-8, **unused_params):
num_layers = FLAGS.multiscale_cnn_lstm_layers
lstm_size = int(FLAGS.lstm_cells)
pool_size=2
num_filters=[256,256,512]
filter_sizes=[1,2,3]
features_size = sum(num_filters)
sub_predictions = []
cnn_input = model_input
cnn_max_frames = model_input.get_shape().as_list()[1]
for layer in range(num_layers):
cnn_output = self.cnn(cnn_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn%d"%(layer+1))
cnn_output_relu = tf.nn.relu(cnn_output)
lstm_memory = self.rnn(cnn_output_relu, lstm_size, num_frames, sub_scope="rnn%d"%(layer+1))
sub_prediction = self.moe(lstm_memory, vocab_size, scopename="moe%d"%(layer+1))
sub_predictions.append(sub_prediction)
cnn_max_frames /= pool_size
max_pooled_cnn_output = tf.reduce_max(
tf.reshape(
cnn_output_relu[:, :cnn_max_frames*2, :],
[-1, cnn_max_frames, pool_size, features_size]
), axis=2)
# for the next cnn layer
cnn_input = max_pooled_cnn_output
num_frames = tf.maximum(num_frames/pool_size, 1)
support_predictions = tf.concat(sub_predictions, axis=1)
predictions = tf.add_n(sub_predictions) / len(sub_predictions)
return {"predictions": predictions,
"support_predictions": support_predictions}
================================================
FILE: youtube-8m-wangheda/all_frame_models/positional_cnn_deep_combine_chain_model.py
================================================
import math
import models
import tensorflow as tf
import numpy as np
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class PositionalCnnDeepCombineChainModel(models.BaseModel):
"""Add positional embedding for cnn"""
def cnn(self,
model_input,
l2_penalty=1e-8,
num_filters = [1024, 1024, 1024],
filter_sizes = [1,2,3],
sub_scope="",
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
shift_inputs = []
for i in xrange(max(filter_sizes)):
if i == 0:
shift_inputs.append(model_input)
else:
shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:])
cnn_outputs = []
for nf, fs in zip(num_filters, filter_sizes):
sub_input = tf.concat(shift_inputs[:fs], axis=2)
sub_filter = tf.get_variable(sub_scope+"cnn-filter-len%d"%fs,
shape=[num_features*fs, nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter))
cnn_output = tf.concat(cnn_outputs, axis=2)
return cnn_output
def add_positional_embedding(self, model_input, num_frames, l2_penalty=1e-8):
batch_size, max_frames, num_features = model_input.get_shape().as_list()
positional_embedding = tf.get_variable("positional_embedding", dtype=tf.float32,
shape=[1, max_frames, num_features],
initializer=tf.zeros_initializer(),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
mask = tf.sequence_mask(lengths=num_frames, maxlen=max_frames, dtype=tf.float32)
model_input_with_positional_embedding = tf.einsum("ijk,ij->ijk", model_input + positional_embedding, mask)
return model_input_with_positional_embedding
def gated_linear_unit(self, input1, input2):
return tf.nn.softmax(input1) * input2
def create_model(self, model_input, vocab_size, num_frames, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", original_input=None, **unused_params):
num_supports = FLAGS.num_supports
num_layers = FLAGS.deep_chain_layers
relu_cells = FLAGS.deep_chain_relu_cells
max_frames = model_input.get_shape().as_list()[1]
relu_layers = []
support_predictions = []
# add positional embedding
model_input = self.add_positional_embedding(model_input, num_frames, l2_penalty=l2_penalty)
mask = tf.sequence_mask(lengths=num_frames, maxlen=max_frames, dtype=tf.float32)
mean_input = tf.einsum("ijk,ij->ik", model_input, mask) \
/ tf.expand_dims(tf.cast(num_frames, dtype=tf.float32), dim=1)
mean_relu = slim.fully_connected(
mean_input,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"mean-relu")
mean_relu_norm = tf.nn.l2_normalize(mean_relu, dim=1)
relu_layers.append(mean_relu_norm)
cnn_output1 = self.cnn(model_input, num_filters=[relu_cells,2*relu_cells,relu_cells], filter_sizes=[1,2,3], sub_scope=sub_scope+"cnn0-output1")
cnn_output2 = self.cnn(model_input, num_filters=[relu_cells,2*relu_cells,relu_cells], filter_sizes=[1,2,3], sub_scope=sub_scope+"cnn0-output2")
cnn_output = self.gated_linear_unit(cnn_output1, cnn_output2)
max_cnn_output = tf.reduce_max(cnn_output, axis=1)
normalized_cnn_output = tf.nn.l2_normalize(max_cnn_output, dim=1)
next_input = normalized_cnn_output
for layer in xrange(num_layers):
sub_prediction = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"prediction-%d"%layer)
support_predictions.append(sub_prediction)
sub_relu = slim.fully_connected(
sub_prediction,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"relu-%d"%layer)
relu_norm = tf.nn.l2_normalize(sub_relu, dim=1)
relu_layers.append(relu_norm)
cnn_output1 = self.cnn(model_input, num_filters=[relu_cells,2*relu_cells,relu_cells], filter_sizes=[1,2,3], sub_scope=sub_scope+"cnn%d-output1"%(layer+1))
cnn_output2 = self.cnn(model_input, num_filters=[relu_cells,2*relu_cells,relu_cells], filter_sizes=[1,2,3], sub_scope=sub_scope+"cnn%d-output2"%(layer+1))
cnn_output = self.gated_linear_unit(cnn_output1, cnn_output2)
max_cnn_output = tf.reduce_max(cnn_output, axis=1)
normalized_cnn_output = tf.nn.l2_normalize(max_cnn_output, dim=1)
next_input = tf.concat([mean_input, normalized_cnn_output] + relu_layers, axis=1)
main_predictions = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"-main")
support_predictions = tf.concat(support_predictions, axis=1)
return {"predictions": main_predictions, "support_predictions": support_predictions}
def sub_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", **unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
def get_mask(self, max_frames, num_frames):
mask_array = []
for i in xrange(max_frames + 1):
tmp = [0.0] * max_frames
for j in xrange(i):
tmp[j] = 1.0
mask_array.append(tmp)
mask_array = np.array(mask_array)
mask_init = tf.constant_initializer(mask_array)
mask_emb = tf.get_variable("mask_emb", shape = [max_frames + 1, max_frames],
dtype = tf.float32, trainable = False, initializer = mask_init)
mask = tf.nn.embedding_lookup(mask_emb, num_frames)
return mask
================================================
FILE: youtube-8m-wangheda/all_frame_models/progressive_attention_lstm_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class ProgressiveAttentionLstmModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = int(FLAGS.lstm_cells)
number_of_layers = FLAGS.lstm_layers
num_attentions = FLAGS.num_attentions
print model_input.get_shape().as_list()
max_frames = model_input.get_shape().as_list()[1]
## Batch normalize the input
stacked_cell = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
att_cell = tf.contrib.rnn.AttentionCellWrapper(cell = tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True),
attn_length=1, state_is_tuple=True)
loss = 0.0
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_cell, model_input,
sequence_length=num_frames,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
final_memory = tf.concat(map(lambda x: x.c, state), axis = 1)
with tf.variable_scope("ATT"):
att_outputs, att_state = tf.nn.dynamic_rnn(att_cell, outputs,
sequence_length=tf.ones_like(num_frames, dtype=tf.int32)*num_attentions,
swap_memory=FLAGS.rnn_swap_memory,
dtype=tf.float32)
print att_outputs
print att_state
att_state, _, _ = att_state
print att_state
att_final_memory = att_state.c
final_state = tf.concat([att_final_memory, final_memory], axis = 1)
print "final_state", final_state
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
predictions = aggregated_model().create_model(
model_input=final_state,
original_input=model_input,
vocab_size=vocab_size,
**unused_params)
print predictions
return predictions
================================================
FILE: youtube-8m-wangheda/all_frame_models/wide_and_deep_model.py
================================================
import sys
import models
import model_utils
import math
import numpy as np
import video_level_models
import frame_level_models
import tensorflow as tf
import utils
import tensorflow.contrib.slim as slim
from tensorflow import flags
FLAGS = flags.FLAGS
class WideAndDeepModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
"""
A super model that combine one or more models
"""
models = FLAGS.wide_and_deep_models
outputs = []
for model_name in map(lambda x: x.strip(), models.split(",")):
model = getattr(frame_level_models, model_name, None)()
output = model.create_model(model_input, vocab_size, num_frames, l2_penalty=l2_penalty, **unused_params)["predictions"]
outputs.append(tf.expand_dims(output, axis=2))
num_models = len(outputs)
model_outputs = tf.concat(outputs, axis=2)
# linear_combination = tf.get_variable("combine", shape=[vocab_size,num_models],
# dtype=tf.float32, initializer=tf.zeros_initializer(),
# regularizer=slim.l2_regularizer(l2_penalty))
# combination = tf.nn.softmax(linear_combination)
combination = tf.fill(dims=[vocab_size,num_models], value=1.0/num_models)
output_sum = tf.einsum("ijk,jk->ij", model_outputs, combination)
return {"predictions": output_sum}
================================================
FILE: youtube-8m-wangheda/all_video_models/.vimrc
================================================
set tabstop=2
set shiftwidth=2
set expandtab
set autoindent
================================================
FILE: youtube-8m-wangheda/all_video_models/__init__.py
================================================
from logistic_model import *
from moe_model import *
from mlp_moe_model import *
from chain_moe_model import *
from deep_chain_model import *
from deep_combine_chain_model import *
from distillchain_deep_combine_chain_model import *
from hidden_chain_model import *
from hidden_combine_chain_model import *
from chain_main_relu_moe_model import *
from chain_support_relu_moe_model import *
from shortcut_chain_support_relu_moe_model import *
from multitask_moe_model import *
from stage2_logistic_model import *
from multitask_divergence_moe_model import *
from multitask_divergence_deep_combine_chain_model import *
================================================
FILE: youtube-8m-wangheda/all_video_models/chain_main_relu_moe_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class ChainMainReluMoeModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", original_input=None, **unused_params):
num_supports = FLAGS.num_supports
input_size = model_input.shape.as_list()[1]
support_predictions = self.sub_model(model_input, num_supports, sub_scope=sub_scope+"-support")
main_relu = slim.fully_connected(
model_input,
input_size,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="main-relu-"+sub_scope)
main_input = tf.concat([main_relu, support_predictions], axis=1)
main_predictions = self.sub_model(main_input, vocab_size, sub_scope=sub_scope+"-main")
return {"predictions": main_predictions, "support_predictions": support_predictions}
def sub_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", **unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
================================================
FILE: youtube-8m-wangheda/all_video_models/chain_moe_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class ChainMoeModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", original_input=None, **unused_params):
num_supports = FLAGS.num_supports
support_predictions = self.sub_model(model_input, num_supports, sub_scope=sub_scope+"-support")
main_input = tf.concat([model_input, support_predictions], axis=1)
main_predictions = self.sub_model(main_input, vocab_size, sub_scope=sub_scope+"-main")
return {"predictions": main_predictions, "support_predictions": support_predictions}
def sub_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", **unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
================================================
FILE: youtube-8m-wangheda/all_video_models/chain_support_relu_moe_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class ChainSupportReluMoeModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", original_input=None, **unused_params):
num_supports = FLAGS.num_supports
input_size = model_input.shape.as_list()[1]
support_relu = slim.fully_connected(
model_input,
input_size,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="main-relu-"+sub_scope)
support_predictions = self.sub_model(support_relu, num_supports, sub_scope=sub_scope+"-support")
main_input = tf.concat([model_input, support_predictions], axis=1)
main_predictions = self.sub_model(main_input, vocab_size, sub_scope=sub_scope+"-main")
return {"predictions": main_predictions, "support_predictions": support_predictions}
def sub_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", **unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
================================================
FILE: youtube-8m-wangheda/all_video_models/deep_chain_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class DeepChainModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", original_input=None,
num_frames=None, **unused_params):
num_supports = FLAGS.num_supports
num_layers = FLAGS.deep_chain_layers
relu_cells = FLAGS.deep_chain_relu_cells
use_length = FLAGS.deep_chain_use_length
if use_length:
model_input = tf.concat([model_input, self.get_length_code(num_frames)], axis=1)
next_input = model_input
support_predictions = []
for layer in xrange(num_layers):
sub_prediction = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"prediction-%d"%layer)
sub_relu = slim.fully_connected(
sub_prediction,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"relu-%d"%layer)
relu_norm = tf.nn.l2_normalize(sub_relu, dim=1)
next_input = tf.concat([model_input, relu_norm], axis=1)
support_predictions.append(sub_prediction)
main_predictions = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"-main")
support_predictions = tf.concat(support_predictions, axis=1)
return {"predictions": main_predictions, "support_predictions": support_predictions}
def sub_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", **unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
def get_length_code(self, num_frames):
code_0 = tf.cast(num_frames <= 60, dtype=tf.int32)
code_1 = tf.cast(num_frames > 60, dtype=tf.int32) * tf.cast(num_frames <= 120, dtype=tf.int32)
code_2 = tf.cast(num_frames > 120, dtype=tf.int32) * tf.cast(num_frames <= 180, dtype=tf.int32)
code_3 = tf.cast(num_frames > 180, dtype=tf.int32) * tf.cast(num_frames <= 240, dtype=tf.int32)
code_4 = tf.cast(num_frames > 240, dtype=tf.int32)
codes = map(lambda x: tf.expand_dims(x, dim=1), [code_0, code_1, code_2, code_3, code_4])
length_code = tf.cast(tf.concat(codes, axis=1), dtype=tf.float32)
return length_code
================================================
FILE: youtube-8m-wangheda/all_video_models/deep_combine_chain_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class DeepCombineChainModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", original_input=None,
dropout=False, keep_prob=None, noise_level=None,
num_frames=None,
**unused_params):
num_supports = FLAGS.num_supports
num_layers = FLAGS.deep_chain_layers
relu_cells = FLAGS.deep_chain_relu_cells
relu_type = FLAGS.deep_chain_relu_type
use_length = FLAGS.deep_chain_use_length
next_input = model_input
support_predictions = []
for layer in xrange(num_layers):
sub_prediction = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"prediction-%d"%layer, dropout=dropout, keep_prob=keep_prob, noise_level=noise_level)
sub_activation = slim.fully_connected(
sub_prediction,
relu_cells,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"relu-%d"%layer)
if relu_type == "elu":
sub_relu = tf.nn.elu(sub_activation)
else: # default: relu
sub_relu = tf.nn.relu(sub_activation)
if noise_level is not None:
print "adding noise to sub_relu, level = ", noise_level
sub_relu = sub_relu + tf.random_normal(tf.shape(sub_relu), mean=0.0, stddev=noise_level)
relu_norm = tf.nn.l2_normalize(sub_relu, dim=1)
next_input = tf.concat([next_input, relu_norm], axis=1)
support_predictions.append(sub_prediction)
main_predictions = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"-main")
support_predictions = tf.concat(support_predictions, axis=1)
return {"predictions": main_predictions, "support_predictions": support_predictions}
def sub_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="",
dropout=False, keep_prob=None, noise_level=None,
**unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
if dropout:
model_input = tf.nn.dropout(model_input, keep_prob=keep_prob)
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
================================================
FILE: youtube-8m-wangheda/all_video_models/distillchain_deep_combine_chain_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class DistillchainDeepCombineChainModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", original_input=None,
dropout=False, keep_prob=None, noise_level=None,
distillation_predictions=None,
num_frames=None,
**unused_params):
assert distillation_predictions is not None, "distillation feature must be used"
num_supports = FLAGS.num_supports
num_layers = FLAGS.deep_chain_layers
relu_cells = FLAGS.deep_chain_relu_cells
relu_type = FLAGS.deep_chain_relu_type
use_length = FLAGS.deep_chain_use_length
distill_relu = slim.fully_connected(
distillation_predictions,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"distillrelu")
distill_norm = tf.nn.l2_normalize(distill_relu, dim=1)
next_input = tf.concat([model_input, distill_norm], axis=1)
support_predictions = []
for layer in xrange(num_layers):
sub_prediction = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"prediction-%d"%layer, dropout=dropout, keep_prob=keep_prob, noise_level=noise_level)
sub_activation = slim.fully_connected(
sub_prediction,
relu_cells,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"relu-%d"%layer)
if relu_type == "elu":
sub_relu = tf.nn.elu(sub_activation)
else: # default: relu
sub_relu = tf.nn.relu(sub_activation)
if noise_level is not None:
print "adding noise to sub_relu, level = ", noise_level
sub_relu = sub_relu + tf.random_normal(tf.shape(sub_relu), mean=0.0, stddev=noise_level)
relu_norm = tf.nn.l2_normalize(sub_relu, dim=1)
next_input = tf.concat([next_input, relu_norm], axis=1)
support_predictions.append(sub_prediction)
main_predictions = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"-main")
support_predictions = tf.concat(support_predictions, axis=1)
return {"predictions": main_predictions, "support_predictions": support_predictions}
def sub_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="",
dropout=False, keep_prob=None, noise_level=None,
**unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
if dropout:
model_input = tf.nn.dropout(model_input, keep_prob=keep_prob)
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
================================================
FILE: youtube-8m-wangheda/all_video_models/hidden_chain_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class HiddenChainModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", original_input=None, **unused_params):
num_supports = FLAGS.num_supports
num_layers = FLAGS.hidden_chain_layers
relu_cells = FLAGS.hidden_chain_relu_cells
next_input = model_input
support_predictions = []
for layer in xrange(num_layers):
sub_relu = slim.fully_connected(
next_input,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"relu-%d"%layer)
sub_prediction = self.sub_model(sub_relu, vocab_size, sub_scope=sub_scope+"prediction-%d"%layer)
relu_norm = tf.nn.l2_normalize(sub_relu, dim=1)
next_input = tf.concat([model_input, relu_norm], axis=1)
support_predictions.append(sub_prediction)
main_predictions = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"-main")
support_predictions = tf.concat(support_predictions, axis=1)
return {"predictions": main_predictions, "support_predictions": support_predictions}
def sub_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", **unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
================================================
FILE: youtube-8m-wangheda/all_video_models/hidden_combine_chain_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class HiddenCombineChainModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", original_input=None, **unused_params):
num_supports = FLAGS.num_supports
num_layers = FLAGS.hidden_chain_layers
relu_cells = FLAGS.hidden_chain_relu_cells
next_input = model_input
support_predictions = []
for layer in xrange(num_layers):
sub_relu = slim.fully_connected(
next_input,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"relu-%d"%layer)
sub_prediction = self.sub_model(sub_relu, vocab_size, sub_scope=sub_scope+"prediction-%d"%layer)
relu_norm = tf.nn.l2_normalize(sub_relu, dim=1)
next_input = tf.concat([next_input, relu_norm], axis=1)
support_predictions.append(sub_prediction)
main_predictions = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"-main")
support_predictions = tf.concat(support_predictions, axis=1)
return {"predictions": main_predictions, "support_predictions": support_predictions}
def sub_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", **unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
================================================
FILE: youtube-8m-wangheda/all_video_models/logistic_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class LogisticModel(models.BaseModel):
"""Logistic model with L2 regularization."""
def create_model(self, model_input, vocab_size, l2_penalty=1e-8, original_input=None, **unused_params):
"""Creates a logistic model.
Args:
model_input: 'batch' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes."""
output = slim.fully_connected(
model_input, vocab_size, activation_fn=tf.nn.sigmoid,
weights_regularizer=slim.l2_regularizer(l2_penalty))
return {"predictions": output}
================================================
FILE: youtube-8m-wangheda/all_video_models/mlp_moe_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class MlpMoeModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self,
model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
sub_scope="",
original_input=None,
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
feature_dim = len(model_input.get_shape().as_list()) - 1
num_features = model_input.get_shape().as_list()[feature_dim]
layer1_size = 2048
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+sub_scope)
expert_activations = slim.fully_connected(
slim.fully_connected(
model_input,
num_features,
activation_fn=tf.nn.relu,
weights_initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
biases_initializer=tf.zeros_initializer(),
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="relu"+sub_scope),
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return {"predictions": final_probabilities}
================================================
FILE: youtube-8m-wangheda/all_video_models/moe_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class MoeModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self,
model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
sub_scope="",
original_input=None,
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return {"predictions": final_probabilities}
================================================
FILE: youtube-8m-wangheda/all_video_models/multitask_divergence_deep_combine_chain_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class MultiTaskDivergenceDeepCombineChainModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="ddcc", original_input=None,
dropout=False, keep_prob=None, noise_level=None,
num_frames=None, **unused_params):
num_supports = FLAGS.num_supports
num_models = FLAGS.divergence_model_count
support_predictions = []
for i in xrange(num_models):
sub_prediction = self.sub_chain_model(model_input,vocab_size, num_mixtures,
l2_penalty, sub_scope+"%d"%i, original_input,
dropout, keep_prob, noise_level)
support_predictions.append(sub_prediction)
support_predictions = tf.stack(support_predictions, axis=1)
main_predictions = tf.reduce_mean(support_predictions, axis=1)
return {"predictions": main_predictions, "support_predictions": support_predictions}
def sub_chain_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", original_input=None,
dropout=False, keep_prob=None, noise_level=None,
num_frames=None,
**unused_params):
num_supports = FLAGS.num_supports
num_layers = FLAGS.deep_chain_layers
relu_cells = FLAGS.deep_chain_relu_cells
relu_type = FLAGS.deep_chain_relu_type
use_length = FLAGS.deep_chain_use_length
input_list = [model_input]
mean_activation = slim.fully_connected(
model_input,
relu_cells,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"relu-mean")
mean_relu = tf.nn.relu(mean_activation)
if noise_level is not None:
print "adding noise to sub_relu, level = ", noise_level
mean_relu = mean_relu + tf.random_normal(tf.shape(mean_relu), mean=0.0, stddev=noise_level)
mean_relu_norm = tf.nn.l2_normalize(mean_relu, dim=1)
next_input = mean_relu_norm
support_predictions = []
for layer in xrange(num_layers):
sub_prediction = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"prediction-%d"%layer, dropout=dropout, keep_prob=keep_prob, noise_level=noise_level)
sub_activation = slim.fully_connected(
sub_prediction,
relu_cells,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"relu-%d"%layer)
sub_relu = tf.nn.relu(sub_activation)
if noise_level is not None:
print "adding noise to sub_relu, level = ", noise_level
sub_relu = sub_relu + tf.random_normal(tf.shape(sub_relu), mean=0.0, stddev=noise_level)
relu_norm = tf.nn.l2_normalize(sub_relu, dim=1)
next_input = tf.concat([next_input, relu_norm], axis=1)
support_predictions.append(sub_prediction)
main_predictions = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"-main")
support_predictions = tf.reduce_mean(tf.stack(support_predictions, axis=0), axis=0)
ratio = min(FLAGS.support_loss_percent, 0.2)
final_predictions = main_predictions * (1.0 - ratio) + support_predictions * ratio
return final_predictions
def sub_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="",
dropout=False, keep_prob=None, noise_level=None,
**unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
if dropout:
model_input = tf.nn.dropout(model_input, keep_prob=keep_prob)
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
================================================
FILE: youtube-8m-wangheda/all_video_models/multitask_divergence_moe_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class MultiTaskDivergenceMoeModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="ddcc", original_input=None,
dropout=False, keep_prob=None, noise_level=None,
num_frames=None, **unused_params):
num_supports = FLAGS.num_supports
num_models = FLAGS.divergence_model_count
support_predictions = []
for i in xrange(num_models):
sub_prediction = self.sub_model(model_input,vocab_size, num_mixtures,
l2_penalty, sub_scope+"%d"%i,
dropout, keep_prob, noise_level)
support_predictions.append(sub_prediction)
support_predictions = tf.stack(support_predictions, axis=1)
main_predictions = tf.reduce_mean(support_predictions, axis=1)
return {"predictions": main_predictions, "support_predictions": support_predictions}
def sub_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="",
dropout=False, keep_prob=None, noise_level=None,
**unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
if dropout:
model_input = tf.nn.dropout(model_input, keep_prob=keep_prob)
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
================================================
FILE: youtube-8m-wangheda/all_video_models/multitask_moe_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class MultiTaskMoeModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", original_input=None, **unused_params):
num_supports = FLAGS.num_supports
support_predictions = self.sub_model(model_input, num_supports, sub_scope=sub_scope+"-support")
main_predictions = self.sub_model(model_input, vocab_size, sub_scope=sub_scope+"-main")
return {"predictions": main_predictions, "support_predictions": support_predictions}
def sub_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", **unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
================================================
FILE: youtube-8m-wangheda/all_video_models/shortcut_chain_support_relu_moe_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class ShortcutChainSupportReluMoeModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", original_input=None, **unused_params):
num_supports = FLAGS.num_supports
input_size = model_input.shape.as_list()[1]
support_input = tf.reduce_mean(original_input, axis=1)
support_relu = slim.fully_connected(
support_input,
input_size,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="main-relu-"+sub_scope)
support_predictions = self.sub_model(support_relu, num_supports, sub_scope=sub_scope+"-support")
main_input = tf.concat([model_input, support_predictions], axis=1)
main_predictions = self.sub_model(main_input, vocab_size, sub_scope=sub_scope+"-main")
return {"predictions": main_predictions, "support_predictions": support_predictions}
def sub_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", **unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
================================================
FILE: youtube-8m-wangheda/all_video_models/stage2_logistic_model.py
================================================
import math
import models
import tensorflow as tf
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
class Stage2LogisticModel(models.BaseModel):
def create_model(self, model_input, vocab_size, l2_penalty=1e-8, original_input=None, **unused_params):
output = tf.nn.sigmoid(model_input + slim.fully_connected(
model_input, vocab_size, activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty)))
return {"predictions": output}
================================================
FILE: youtube-8m-wangheda/average_precision_calculator.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Calculate or keep track of the interpolated average precision.
It provides an interface for calculating interpolated average precision for an
entire list or the top-n ranked items. For the definition of the
(non-)interpolated average precision:
http://trec.nist.gov/pubs/trec15/appendices/CE.MEASURES06.pdf
Example usages:
1) Use it as a static function call to directly calculate average precision for
a short ranked list in the memory.
```
import random
p = np.array([random.random() for _ in xrange(10)])
a = np.array([random.choice([0, 1]) for _ in xrange(10)])
ap = average_precision_calculator.AveragePrecisionCalculator.ap(p, a)
```
2) Use it as an object for long ranked list that cannot be stored in memory or
the case where partial predictions can be observed at a time (Tensorflow
predictions). In this case, we first call the function accumulate many times
to process parts of the ranked list. After processing all the parts, we call
peek_interpolated_ap_at_n.
```
p1 = np.array([random.random() for _ in xrange(5)])
a1 = np.array([random.choice([0, 1]) for _ in xrange(5)])
p2 = np.array([random.random() for _ in xrange(5)])
a2 = np.array([random.choice([0, 1]) for _ in xrange(5)])
# interpolated average precision at 10 using 1000 break points
calculator = average_precision_calculator.AveragePrecisionCalculator(10)
calculator.accumulate(p1, a1)
calculator.accumulate(p2, a2)
ap3 = calculator.peek_ap_at_n()
```
"""
import heapq
import random
import numbers
import numpy
class AveragePrecisionCalculator(object):
"""Calculate the average precision and average precision at n."""
def __init__(self, top_n=None):
"""Construct an AveragePrecisionCalculator to calculate average precision.
This class is used to calculate the average precision for a single label.
Args:
top_n: A positive Integer specifying the average precision at n, or
None to use all provided data points.
Raises:
ValueError: An error occurred when the top_n is not a positive integer.
"""
if not ((isinstance(top_n, int) and top_n >= 0) or top_n is None):
raise ValueError("top_n must be a positive integer or None.")
self._top_n = top_n # average precision at n
self._total_positives = 0 # total number of positives have seen
self._heap = [] # max heap of (prediction, actual)
@property
def heap_size(self):
"""Gets the heap size maintained in the class."""
return len(self._heap)
@property
def num_accumulated_positives(self):
"""Gets the number of positive samples that have been accumulated."""
return self._total_positives
def accumulate(self, predictions, actuals, num_positives=None):
"""Accumulate the predictions and their ground truth labels.
After the function call, we may call peek_ap_at_n to actually calculate
the average precision.
Note predictions and actuals must have the same shape.
Args:
predictions: a list storing the prediction scores.
actuals: a list storing the ground truth labels. Any value
larger than 0 will be treated as positives, otherwise as negatives.
num_positives = If the 'predictions' and 'actuals' inputs aren't complete,
then it's possible some true positives were missed in them. In that case,
you can provide 'num_positives' in order to accurately track recall.
Raises:
ValueError: An error occurred when the format of the input is not the
numpy 1-D array or the shape of predictions and actuals does not match.
"""
if len(predictions) != len(actuals):
raise ValueError("the shape of predictions and actuals does not match.")
if not num_positives is None:
if not isinstance(num_positives, numbers.Number) or num_positives < 0:
raise ValueError("'num_positives' was provided but it wan't a nonzero number.")
if not num_positives is None:
self._total_positives += num_positives
else:
self._total_positives += numpy.size(numpy.where(actuals > 0))
topk = self._top_n
heap = self._heap
for i in range(numpy.size(predictions)):
if topk is None or len(heap) < topk:
heapq.heappush(heap, (predictions[i], actuals[i]))
else:
if predictions[i] > heap[0][0]: # heap[0] is the smallest
heapq.heappop(heap)
heapq.heappush(heap, (predictions[i], actuals[i]))
def clear(self):
"""Clear the accumulated predictions."""
self._heap = []
self._total_positives = 0
def peek_ap_at_n(self):
"""Peek the non-interpolated average precision at n.
Returns:
The non-interpolated average precision at n (default 0).
If n is larger than the length of the ranked list,
the average precision will be returned.
"""
if self.heap_size <= 0:
return 0
predlists = numpy.array(list(zip(*self._heap)))
ap = self.ap_at_n(predlists[0],
predlists[1],
n=self._top_n,
total_num_positives=self._total_positives)
return ap
@staticmethod
def ap(predictions, actuals):
"""Calculate the non-interpolated average precision.
Args:
predictions: a numpy 1-D array storing the sparse prediction scores.
actuals: a numpy 1-D array storing the ground truth labels. Any value
larger than 0 will be treated as positives, otherwise as negatives.
Returns:
The non-interpolated average precision at n.
If n is larger than the length of the ranked list,
the average precision will be returned.
Raises:
ValueError: An error occurred when the format of the input is not the
numpy 1-D array or the shape of predictions and actuals does not match.
"""
return AveragePrecisionCalculator.ap_at_n(predictions,
actuals,
n=None)
@staticmethod
def ap_at_n(predictions, actuals, n=20, total_num_positives=None):
"""Calculate the non-interpolated average precision.
Args:
predictions: a numpy 1-D array storing the sparse prediction scores.
actuals: a numpy 1-D array storing the ground truth labels. Any value
larger than 0 will be treated as positives, otherwise as negatives.
n: the top n items to be considered in ap@n.
total_num_positives : (optionally) you can specify the number of total
positive
in the list. If specified, it will be used in calculation.
Returns:
The non-interpolated average precision at n.
If n is larger than the length of the ranked list,
the average precision will be returned.
Raises:
ValueError: An error occurred when
1) the format of the input is not the numpy 1-D array;
2) the shape of predictions and actuals does not match;
3) the input n is not a positive integer.
"""
if len(predictions) != len(actuals):
raise ValueError("the shape of predictions and actuals does not match.")
if n is not None:
if not isinstance(n, int) or n <= 0:
raise ValueError("n must be 'None' or a positive integer."
" It was '%s'." % n)
ap = 0.0
predictions = numpy.array(predictions)
actuals = numpy.array(actuals)
# add a shuffler to avoid overestimating the ap
predictions, actuals = AveragePrecisionCalculator._shuffle(predictions,
actuals)
sortidx = sorted(
range(len(predictions)),
key=lambda k: predictions[k],
reverse=True)
if total_num_positives is None:
numpos = numpy.size(numpy.where(actuals > 0))
else:
numpos = total_num_positives
if numpos == 0:
return 0
if n is not None:
numpos = min(numpos, n)
delta_recall = 1.0 / numpos
poscount = 0.0
# calculate the ap
r = len(sortidx)
if n is not None:
r = min(r, n)
for i in range(r):
if actuals[sortidx[i]] > 0:
poscount += 1
ap += poscount / (i + 1) * delta_recall
return ap
@staticmethod
def _shuffle(predictions, actuals):
random.seed(0)
suffidx = random.sample(range(len(predictions)), len(predictions))
predictions = predictions[suffidx]
actuals = actuals[suffidx]
return predictions, actuals
@staticmethod
def _zero_one_normalize(predictions, epsilon=1e-7):
"""Normalize the predictions to the range between 0.0 and 1.0.
For some predictions like SVM predictions, we need to normalize them before
calculate the interpolated average precision. The normalization will not
change the rank in the original list and thus won't change the average
precision.
Args:
predictions: a numpy 1-D array storing the sparse prediction scores.
epsilon: a small constant to avoid denominator being zero.
Returns:
The normalized prediction.
"""
denominator = numpy.max(predictions) - numpy.min(predictions)
ret = (predictions - numpy.min(predictions)) / numpy.max(denominator,
epsilon)
return ret
================================================
FILE: youtube-8m-wangheda/bagging_scripts/cnn-deep-combine-chain-bagging.sh
================================================
#!/bin/bash
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
# base_model or sub_model_1 or sub_model_2 or so on
model_type="$1"
model_name="cnn_deep_combine_chain_bagging"
MODEL_DIR="../model/${model_name}"
vocab_file="resources/train.video_id.vocab"
default_freq_file="resources/train.video_id.freq"
if [ ! -f $vocab_file ]; then
cd resources
wget http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv
echo "OOV" > train.video_id.vocab
cat train_labels.csv | cut -d ',' -f 1 >> train.video_id.vocab
cd ..
fi
vocab_checksum=$(md5sum $vocab_file | cut -d ' ' -f 1)
if [ "$vocab_checksum" == "b74b8f2592cad5dd21bf614d1438db98" ]; then
echo $vocab_file is valid
else
echo $vocab_file is corrupted
exit 1
fi
if [ ! -f $default_freq_file ]; then
cat $vocab_file | awk '{print 1}' > $default_freq_file
fi
base_model_dir="${MODEL_DIR}/base_model"
if [ $model_type == "base_model" ]; then
# base model
rm ${MODEL_DIR}/ensemble.conf
mkdir -p $base_model_dir
for j in 1 2; do
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="$base_model_dir" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--reweight=True \
--sample_vocab_file="$vocab_file" \
--sample_freq_file="$default_freq_file" \
--model=CnnDeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_layers=4 \
--deep_chain_relu_cells=128 \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label,label" \
--support_loss_percent=0.05 \
--batch_size=128 \
--base_learning_rate=0.001 \
--num_readers=2 \
--num_epochs=2 \
--keep_checkpoint_every_n_hour=72.0
done
elif [[ $model_type =~ ^sub_model ]]; then
# sub model
sub_model_dir="${MODEL_DIR}/${model_type}"
if [ ! -d $sub_model_dir ]; then
cp -r $base_model_dir $sub_model_dir
# generate freq file
python training_utils/sample_freq.py \
--video_id_file="$vocab_file" \
--output_freq_file="${sub_model_dir}/train.video_id.freq"
# train N models with re-weighted samples
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="$sub_model_dir" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--reweight=True \
--sample_vocab_file="$vocab_file" \
--sample_freq_file="$default_freq_file" \
--model=CnnDeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_layers=4 \
--deep_chain_relu_cells=128 \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label,label" \
--support_loss_percent=0.05 \
--batch_size=128 \
--base_learning_rate=0.001 \
--num_readers=2 \
--num_epochs=2 \
--keep_checkpoint_every_n_hour=72.0
fi
# inference-pre-ensemble
for part in test ensemble_validate ensemble_train; do
output_dir="/Youtube-8M/model_predictions/${part}/${model_name}/${model_type}"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-pre-ensemble.py \
--output_dir=$output_dir \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=CnnDeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_layers=4 \
--deep_chain_relu_cells=128 \
--batch_size=32 \
--file_size=4096
fi
done
echo "${model_name}/${model_type}" >> ${MODEL_DIR}/ensemble.conf
fi
# on ensemble server
#cd ../youtube-8m-ensemble
#bash ensemble_scripts/eval-mean_model.sh ${model_name}/ensemble_mean_model ${MODEL_DIR}/ensemble.conf
#bash ensemble_scripts/infer-mean_model.sh ${model_name}/ensemble_mean_model ${MODEL_DIR}/ensemble.conf
================================================
FILE: youtube-8m-wangheda/bagging_scripts/distillation-video-dcc-bagging.sh
================================================
#!/bin/bash
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
model_name="distillation_video_dcc_bagging"
MODEL_DIR="../model/${model_name}"
rm ${MODEL_DIR}/ensemble.conf
vocab_file="resources/train.video_id.vocab"
if [ ! -f $vocab_file ]; then
cd resources
wget http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv
echo "OOV" > train.video_id.vocab
cat train_labels.csv | cut -d ',' -f 1 >> train.video_id.vocab
cd ..
fi
vocab_checksum=$(md5sum $vocab_file | cut -d ' ' -f 1)
if [ "$vocab_checksum" == "b74b8f2592cad5dd21bf614d1438db98" ]; then
echo $vocab_file is valid
else
echo $vocab_file is corrupted
exit 1
fi
if [ ! -f $default_freq_file ]; then
cat $vocab_file | awk '{print 1}' > $default_freq_file
fi
for i in {1..4}; do
sub_model_dir="${MODEL_DIR}/sub_model_${i}"
if [ ! -d $sub_model_dir ]; then
mkdir -p $sub_model_dir
# generate freq file
if [ ! -f ${sub_model_dir}/train.video_id.freq ]; then
python training_utils/sample_freq.py \
--video_id_file="$vocab_file" \
--output_freq_file="${sub_model_dir}/train.video_id.freq"
fi
# train N models with re-weighted samples
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="$sub_model_dir" \
--train_data_pattern="/Youtube-8M/distillation/video/train/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_type=1 \
--distillation_percent=0.5 \
--model=DeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=4 \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_type="label,label,label,label" \
--num_supports=18864 \
--support_loss_percent=0.05 \
--reweight=True \
--sample_vocab_file="resources/train.video_id.vocab" \
--sample_freq_file="${sub_model_dir}/train.video_id.freq" \
--keep_checkpoint_every_n_hour=10 \
--keep_checkpoint_interval=6 \
--base_learning_rate=0.01 \
--data_augmenter=NoiseAugmenter \
--input_noise_level=0.1 \
--num_readers=4 \
--num_epochs=6 \
--batch_size=512
fi
# inference-pre-ensemble
for part in test ensemble_validate ensemble_train; do
output_dir="/Youtube-8M/model_predictions/${part}/${model_name}/sub_model_$i"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-pre-ensemble.py \
--output_dir="$output_dir" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=DeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=4 \
--batch_size=1024 \
--file_size=4096
fi
done
echo "${model_name}/sub_model_$i" >> ${MODEL_DIR}/ensemble.conf
done
#cd ../youtube-8m-ensemble
#bash ensemble_scripts/eval-mean_model.sh video_bagging/ensemble_mean_model ${MODEL_DIR}/ensemble.conf
#bash ensemble_scripts/infer-mean_model.sh video_bagging/ensemble_mean_model ${MODEL_DIR}/ensemble.conf
================================================
FILE: youtube-8m-wangheda/bagging_scripts/lstmattention8max-bagging.sh
================================================
#!/bin/bash
DEFAULT_GPU_ID=1
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
# base_model or sub_model_1 or sub_model_2 or so on
model_type="$1"
model_name="lstmattention8max_bagging"
MODEL_DIR="../model/${model_name}"
vocab_file="resources/train.video_id.vocab"
default_freq_file="resources/train.video_id.freq"
if [ ! -f $vocab_file ]; then
cd resources
wget http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv
echo "OOV" > train.video_id.vocab
cat train_labels.csv | cut -d ',' -f 1 >> train.video_id.vocab
cd ..
fi
vocab_checksum=$(md5sum $vocab_file | cut -d ' ' -f 1)
if [ "$vocab_checksum" == "b74b8f2592cad5dd21bf614d1438db98" ]; then
echo $vocab_file is valid
else
echo $vocab_file is corrupted
exit 1
fi
if [ ! -f $default_freq_file ]; then
cat $vocab_file | awk '{print 1}' > $default_freq_file
fi
base_model_dir="${MODEL_DIR}/base_model"
if [ $model_type == "base_model" ]; then
# base model
rm ${MODEL_DIR}/ensemble.conf
mkdir -p $base_model_dir
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="$base_model_dir" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--reweight=True \
--sample_vocab_file="$vocab_file" \
--sample_freq_file="$default_freq_file" \
--model=LstmAttentionMaxPoolingModel \
--moe_num_mixtures=8 \
--lstm_attentions=8 \
--rnn_swap_memory=True \
--base_learning_rate=0.001 \
--num_readers=2 \
--num_epochs=5 \
--batch_size=128 \
--keep_checkpoint_every_n_hour=0.5
elif [[ $model_type =~ ^sub_model ]]; then
# sub model
sub_model_dir="${MODEL_DIR}/${model_type}"
if [ ! -d $sub_model_dir ]; then
cp -r $base_model_dir $sub_model_dir
# generate freq file
python training_utils/sample_freq.py \
--video_id_file="$vocab_file" \
--output_freq_file="${sub_model_dir}/train.video_id.freq"
# train N models with re-weighted samples
CUDA_VISIBLE_DEVICES="$GPU_ID" python train-with-rebuild.py \
--train_dir="$sub_model_dir" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--reweight=True \
--sample_vocab_file="resources/train.video_id.vocab" \
--sample_freq_file="${sub_model_dir}/train.video_id.freq" \
--model=LstmAttentionMaxPoolingModel \
--moe_num_mixtures=8 \
--lstm_attentions=8 \
--rnn_swap_memory=True \
--base_learning_rate=0.001 \
--num_readers=4 \
--num_epochs=1 \
--batch_size=128 \
--keep_checkpoint_every_n_hour=72.0
fi
# inference-pre-ensemble
for part in test ensemble_validate ensemble_train; do
output_dir="/Youtube-8M/model_predictions/${part}/${model_name}/${model_type}"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-pre-ensemble.py \
--output_dir="$output_dir" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=LstmAttentionMaxPoolingModel \
--moe_num_mixtures=8 \
--lstm_attentions=8 \
--rnn_swap_memory=True \
--batch_size=32 \
--file_size=4096
fi
done
echo "${model_name}/${model_type}" >> ${MODEL_DIR}/ensemble.conf
fi
# on ensemble server
#cd ../youtube-8m-ensemble
#bash ensemble_scripts/eval-mean_model.sh ${model_name}/ensemble_mean_model ${MODEL_DIR}/ensemble.conf
#bash ensemble_scripts/infer-mean_model.sh ${model_name}/ensemble_mean_model ${MODEL_DIR}/ensemble.conf
================================================
FILE: youtube-8m-wangheda/bagging_scripts/lstmparalleloutput-bagging.sh
================================================
#!/bin/bash
DEFAULT_GPU_ID=1
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
# base_model or sub_model_1 or sub_model_2 or so on
model_type="$1"
model_name="lstmparalleloutput_bagging"
MODEL_DIR="../model/${model_name}"
vocab_file="resources/train.video_id.vocab"
default_freq_file="resources/train.video_id.freq"
if [ ! -f $vocab_file ]; then
cd resources
wget http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv
echo "OOV" > train.video_id.vocab
cat train_labels.csv | cut -d ',' -f 1 >> train.video_id.vocab
cd ..
fi
vocab_checksum=$(md5sum $vocab_file | cut -d ' ' -f 1)
if [ "$vocab_checksum" == "b74b8f2592cad5dd21bf614d1438db98" ]; then
echo $vocab_file is valid
else
echo $vocab_file is corrupted
exit 1
fi
if [ ! -f $default_freq_file ]; then
cat $vocab_file | awk '{print 1}' > $default_freq_file
fi
base_model_dir="${MODEL_DIR}/base_model"
if [ $model_type == "base_model" ]; then
# base model
rm ${MODEL_DIR}/ensemble.conf
mkdir -p $base_model_dir
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="$base_model_dir" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--reweight=True \
--sample_vocab_file="$vocab_file" \
--sample_freq_file="$default_freq_file" \
--model=LstmParallelFinaloutputModel \
--lstm_cells="1024,128" \
--moe_num_mixtures=8 \
--rnn_swap_memory=True \
--base_learning_rate=0.0008 \
--num_readers=2 \
--num_epochs=3 \
--batch_size=128 \
--keep_checkpoint_every_n_hour=2.0
elif [[ $model_type =~ ^sub_model ]]; then
# sub model
sub_model_dir="${MODEL_DIR}/${model_type}"
if [ ! -d $sub_model_dir ]; then
cp -r $base_model_dir $sub_model_dir
# generate freq file
python training_utils/sample_freq.py \
--video_id_file="$vocab_file" \
--output_freq_file="${sub_model_dir}/train.video_id.freq"
# train N models with re-weighted samples
CUDA_VISIBLE_DEVICES="$GPU_ID" python train-with-rebuild.py \
--train_dir="$sub_model_dir" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--reweight=True \
--sample_vocab_file="resources/train.video_id.vocab" \
--sample_freq_file="${sub_model_dir}/train.video_id.freq" \
--model=LstmParallelFinaloutputModel \
--lstm_cells="1024,128" \
--moe_num_mixtures=8 \
--rnn_swap_memory=True \
--base_learning_rate=0.0008 \
--num_readers=4 \
--num_epochs=1 \
--batch_size=128 \
--keep_checkpoint_every_n_hour=72.0
fi
# inference-pre-ensemble
for part in test ensemble_validate ensemble_train; do
output_dir="/Youtube-8M/model_predictions/${part}/${model_name}/${model_type}"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-pre-ensemble.py \
--output_dir=$output_dir \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=32 \
--model=LstmParallelFinaloutputModel \
--lstm_cells="1024,128" \
--moe_num_mixtures=8 \
--rnn_swap_memory=True \
--file_size=4096
fi
done
echo "${model_name}/${model_type}" >> ${MODEL_DIR}/ensemble.conf
fi
# on ensemble server
#cd ../youtube-8m-ensemble
#bash ensemble_scripts/eval-mean_model.sh ${model_name}/ensemble_mean_model ${MODEL_DIR}/ensemble.conf
#bash ensemble_scripts/infer-mean_model.sh ${model_name}/ensemble_mean_model ${MODEL_DIR}/ensemble.conf
================================================
FILE: youtube-8m-wangheda/bagging_scripts/video-deep-combine-chain-bagging.sh
================================================
#!/bin/bash
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
MODEL_DIR="../model/video_bagging"
rm ${MODEL_DIR}/ensemble.conf
vocab_file="resources/train.video_id.vocab"
if [ ! -f $vocab_file ]; then
cd resources
wget http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv
echo "OOV" > train.video_id.vocab
cat train_labels.csv | cut -d ',' -f 1 >> train.video_id.vocab
cd ..
fi
vocab_checksum=$(md5sum $vocab_file | cut -d ' ' -f 1)
if [ "$vocab_checksum" == "b74b8f2592cad5dd21bf614d1438db98" ]; then
echo $vocab_file is valid
else
echo $vocab_file is corrupted
exit 1
fi
if [ ! -f $default_freq_file ]; then
cat $vocab_file | awk '{print 1}' > $default_freq_file
fi
for i in {1..8}; do
sub_model_dir="${MODEL_DIR}/sub_model_${i}"
mkdir -p $sub_model_dir
# generate freq file
python training_utils/sample_freq.py \
--video_id_file="$vocab_file" \
--output_freq_file="${sub_model_dir}/train.video_id.freq"
# train N models with re-weighted samples
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="$sub_model_dir" \
--train_data_pattern="/Youtube-8M/data/video/train/train*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=DeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=4 \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label,label" \
--num_supports=18864 \
--support_loss_percent=0.05 \
--reweight=True \
--sample_vocab_file="resources/train.video_id.vocab" \
--sample_freq_file="${sub_model_dir}/train.video_id.freq" \
--keep_checkpoint_every_n_hour=0.25 \
--base_learning_rate=0.01 \
--data_augmenter=NoiseAugmenter \
--input_noise_level=0.2 \
--num_readers=2 \
--num_epochs=5 \
--batch_size=1024
# inference-pre-ensemble
for part in test ensemble_validate ensemble_train; do
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/video_deep_combine_chain_bagging/sub_model_$i" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--file_size=4096
done
echo "video_deep_combine_chain_bagging/sub_model_$i" >> ${MODEL_DIR}/ensemble.conf
done
cd ../youtube-8m-ensemble
CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/eval-mean_model.sh video_bagging/ensemble_mean_model ${MODEL_DIR}/ensemble.conf
CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/infer-mean_model.sh video_bagging/ensemble_mean_model ${MODEL_DIR}/ensemble.conf
================================================
FILE: youtube-8m-wangheda/boosting_scripts/cnn-deep-combine-chain-boosting.sh
================================================
#!/bin/bash
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
# base_model or sub_model_1 or sub_model_2 or so on
model_type="$1"
model_name="cnn_deep_combine_chain_boosting"
MODEL_DIR="../model/${model_name}"
vocab_file="resources/train.video_id.vocab"
default_freq_file="resources/train.video_id.freq"
if [ ! -f $vocab_file ]; then
cd resources
wget http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv
echo "OOV" > train.video_id.vocab
cat train_labels.csv | cut -d ',' -f 1 >> train.video_id.vocab
cd ..
fi
vocab_checksum=$(md5sum $vocab_file | cut -d ' ' -f 1)
if [ "$vocab_checksum" == "b74b8f2592cad5dd21bf614d1438db98" ]; then
echo $vocab_file is valid
else
echo $vocab_file is corrupted
exit 1
fi
if [ ! -f $default_freq_file ]; then
cat $vocab_file | awk '{print 1}' > $default_freq_file
fi
base_model_dir="${MODEL_DIR}/base_model"
if [ $model_type == "base_model" ]; then
# base model
rm ${MODEL_DIR}/ensemble.conf
mkdir -p $base_model_dir
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="$base_model_dir" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--reweight=True \
--sample_vocab_file="$vocab_file" \
--sample_freq_file="$default_freq_file" \
--model=CnnDeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_layers=4 \
--deep_chain_relu_cells=128 \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label,label" \
--support_loss_percent=0.05 \
--batch_size=128 \
--base_learning_rate=0.001 \
--num_readers=2 \
--num_epochs=4 \
--keep_checkpoint_every_n_hour=72.0
elif [[ $model_type =~ ^sub_model ]]; then
last_freq_file=$default_freq_file
# sub model
for i in {1..8}; do
sub_model_dir="${MODEL_DIR}/sub_model_${i}"
if [ ! -d $sub_model_dir ]; then
cp -r $base_model_dir $sub_model_dir
echo "training model #$i, reweighting with $last_freq_file"
# train N models with re-weighted samples
CUDA_VISIBLE_DEVICES="$GPU_ID" python train-with-rebuild.py \
--train_dir="$sub_model_dir" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--reweight=True \
--sample_vocab_file="$vocab_file" \
--sample_freq_file="$last_freq_file" \
--model=CnnDeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_layers=4 \
--deep_chain_relu_cells=128 \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label,label" \
--support_loss_percent=0.05 \
--batch_size=128 \
--base_learning_rate=0.001 \
--num_readers=2 \
--num_epochs=2 \
--keep_checkpoint_every_n_hour=72.0
fi
# get error mapping
sample_error_file="${sub_model_dir}/train.video_id.error"
if [ ! -f $sample_error_file ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-sample-error.py \
--output_file="$sample_error_file" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/frame/train/train*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=CnnDeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_layers=4 \
--deep_chain_relu_cells=128 \
--num_readers=1 \
--batch_size=64
fi
# generate resample freq file
output_freq_file="${sub_model_dir}/train.video_id.next_freq"
if [ ! -f $output_freq_file ]; then
python training_utils/reweight_sample_freq.py \
--clip_weight=5.0 \
--video_id_file="$vocab_file" \
--input_freq_file="$last_freq_file" \
--input_error_file="${sub_model_dir}/train.video_id.error" \
--output_freq_file="$output_freq_file"
fi
last_freq_file="${sub_model_dir}/train.video_id.next_freq"
echo "${model_name}/sub_model_$i" >> ${MODEL_DIR}/ensemble.conf
done
elif [[ $model_type =~ ^ensemble ]]; then
for i in {1..8}; do
sub_model_dir="${MODEL_DIR}/sub_model_${i}"
# inference-pre-ensemble
for part in test ensemble_validate ensemble_train; do
output_dir="/Youtube-8M/model_predictions/${part}/${model_name}/sub_model_${i}"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-pre-ensemble.py \
--output_dir="$output_dir" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=CnnDeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_layers=4 \
--deep_chain_relu_cells=128 \
--batch_size=64 \
--file_size=4096
fi
done
done
# on ensemble server
cd ../youtube-8m-ensemble
CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/train-attention_matrix_model.sh ${model_name}/ensemble_attention_matrix_model ${MODEL_DIR}/ensemble.conf
CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/eval-attention_matrix_model.sh ${model_name}/ensemble_attention_matrix_model ${MODEL_DIR}/ensemble.conf
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/infer-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
fi
================================================
FILE: youtube-8m-wangheda/boosting_scripts/distillation-cnn-dcc-boosting.sh
================================================
#!/bin/bash
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
# base_model or sub_model_1 or sub_model_2 or so on
model_type="$1"
model_name="distillation_cnn_dcc_boosting"
MODEL="CnnDeepCombineChainModel"
MODEL_DIR="../model/${model_name}"
vocab_file="resources/train.video_id.vocab"
default_freq_file="resources/train.video_id.freq"
if [ ! -f $vocab_file ]; then
cd resources
wget http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv
echo "OOV" > train.video_id.vocab
cat train_labels.csv | cut -d ',' -f 1 >> train.video_id.vocab
cd ..
fi
vocab_checksum=$(md5sum $vocab_file | cut -d ' ' -f 1)
if [ "$vocab_checksum" == "b74b8f2592cad5dd21bf614d1438db98" ]; then
echo $vocab_file is valid
else
echo $vocab_file is corrupted
exit 1
fi
if [ ! -f $default_freq_file ]; then
cat $vocab_file | awk '{print 1}' > $default_freq_file
fi
base_model_dir="${MODEL_DIR}/base_model"
if [ $model_type == "base_model" ]; then
# base model
rm ${MODEL_DIR}/ensemble.conf
mkdir -p $base_model_dir
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="$base_model_dir" \
--train_data_pattern="/Youtube-8M/distillation/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_type=1 \
--distillation_percent=0.5 \
--reweight=True \
--sample_vocab_file="$vocab_file" \
--sample_freq_file="$default_freq_file" \
--model=$MODEL \
--moe_num_mixtures=4 \
--deep_chain_layers=4 \
--deep_chain_relu_cells=128 \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label,label" \
--support_loss_percent=0.05 \
--base_learning_rate=0.001 \
--num_readers=2 \
--num_epochs=4 \
--batch_size=96 \
--keep_checkpoint_every_n_hour=72.0
elif [[ $model_type =~ ^sub_model ]]; then
last_freq_file=$default_freq_file
# sub model
for i in {1..3}; do
sub_model_dir="${MODEL_DIR}/sub_model_${i}"
if [ ! -d $sub_model_dir ]; then
cp -r $base_model_dir $sub_model_dir
echo "training model #$i, reweighting with $last_freq_file"
# train N models with re-weighted samples
CUDA_VISIBLE_DEVICES="$GPU_ID" python train-with-rebuild.py \
--train_dir="$sub_model_dir" \
--train_data_pattern="/Youtube-8M/distillation/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_type=1 \
--distillation_percent=0.5 \
--reweight=True \
--sample_vocab_file="resources/train.video_id.vocab" \
--sample_freq_file="$last_freq_file" \
--model=$MODEL \
--moe_num_mixtures=4 \
--deep_chain_layers=4 \
--deep_chain_relu_cells=128 \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label,label" \
--support_loss_percent=0.05 \
--base_learning_rate=0.001 \
--num_readers=2 \
--num_epochs=2 \
--batch_size=96 \
--keep_checkpoint_every_n_hour=72.0
fi
# get error mapping
sample_error_file="${sub_model_dir}/train.video_id.error"
if [ ! -f $sample_error_file ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-sample-error.py \
--output_file="$sample_error_file" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--moe_num_mixtures=4 \
--deep_chain_layers=4 \
--deep_chain_relu_cells=128 \
--batch_size=128
fi
# generate resample freq file
output_freq_file="${sub_model_dir}/train.video_id.next_freq"
if [ ! -f $output_freq_file ]; then
python training_utils/reweight_sample_freq.py \
--clip_weight=5.0 \
--video_id_file="$vocab_file" \
--input_freq_file="$last_freq_file" \
--input_error_file="${sub_model_dir}/train.video_id.error" \
--output_freq_file="$output_freq_file"
fi
last_freq_file="${sub_model_dir}/train.video_id.next_freq"
echo "${model_name}/sub_model_$i" >> ${MODEL_DIR}/ensemble.conf
done
elif [[ $model_type =~ ^ensemble ]]; then
for i in {1..4}; do
sub_model_dir="${MODEL_DIR}/sub_model_${i}"
# inference-pre-ensemble
for part in test ensemble_validate ensemble_train; do
output_dir="/Youtube-8M/model_predictions_local/${part}/${model_name}/sub_model_${i}"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-pre-ensemble.py \
--output_dir="$output_dir" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--moe_num_mixtures=4 \
--deep_chain_layers=4 \
--deep_chain_relu_cells=128 \
--batch_size=64 \
--file_size=4096
fi
done
done
# on ensemble server
cd ../youtube-8m-ensemble
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/train-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/eval-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/preensemble-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
fi
================================================
FILE: youtube-8m-wangheda/boosting_scripts/distillation-lstmcnn-dcc-boosting.sh
================================================
#!/bin/bash
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
# base_model or sub_model_1 or sub_model_2 or so on
model_type="$1"
model_name="distillation_lstmcnn_dcc_boosting"
MODEL="LstmCnnDeepCombineChainModel"
MODEL_DIR="../model/${model_name}"
vocab_file="resources/train.video_id.vocab"
default_freq_file="resources/train.video_id.freq"
if [ ! -f $vocab_file ]; then
cd resources
wget http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv
echo "OOV" > train.video_id.vocab
cat train_labels.csv | cut -d ',' -f 1 >> train.video_id.vocab
cd ..
fi
vocab_checksum=$(md5sum $vocab_file | cut -d ' ' -f 1)
if [ "$vocab_checksum" == "b74b8f2592cad5dd21bf614d1438db98" ]; then
echo $vocab_file is valid
else
echo $vocab_file is corrupted
exit 1
fi
if [ ! -f $default_freq_file ]; then
cat $vocab_file | awk '{print 1}' > $default_freq_file
fi
base_model_dir="${MODEL_DIR}/base_model"
if [ $model_type == "base_model" ]; then
# base model
rm ${MODEL_DIR}/ensemble.conf
mkdir -p $base_model_dir
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="$base_model_dir" \
--train_data_pattern="/Youtube-8M/distillation/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_type=1 \
--distillation_percent=0.5 \
--reweight=True \
--sample_vocab_file="$vocab_file" \
--sample_freq_file="$default_freq_file" \
--model=$MODEL \
--deep_chain_layers=2 \
--deep_chain_relu_cells=128 \
--moe_num_mixtures=4 \
--lstm_layers=1 \
--lstm_cells="1024,128" \
--rnn_swap_memory=True \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_type="label,label" \
--support_loss_percent=0.07 \
--base_learning_rate=0.001 \
--num_readers=4 \
--num_epochs=4 \
--batch_size=128 \
--keep_checkpoint_every_n_hour=6.0
elif [[ $model_type =~ ^sub_model ]]; then
last_freq_file=$default_freq_file
# sub model
for i in {1..4}; do
sub_model_dir="${MODEL_DIR}/sub_model_${i}"
if [ ! -d $sub_model_dir ]; then
cp -r $base_model_dir $sub_model_dir
echo "training model #$i, reweighting with $last_freq_file"
# train N models with re-weighted samples
CUDA_VISIBLE_DEVICES="$GPU_ID" python train-with-rebuild.py \
--train_dir="$sub_model_dir" \
--train_data_pattern="/Youtube-8M/distillation/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_type=1 \
--distillation_percent=0.5 \
--reweight=True \
--sample_vocab_file="resources/train.video_id.vocab" \
--sample_freq_file="$last_freq_file" \
--model=$MODEL \
--deep_chain_layers=2 \
--deep_chain_relu_cells=128 \
--moe_num_mixtures=4 \
--lstm_layers=1 \
--lstm_cells="1024,128" \
--rnn_swap_memory=True \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_type="label,label" \
--support_loss_percent=0.07 \
--base_learning_rate=0.001 \
--num_readers=4 \
--num_epochs=2 \
--batch_size=128 \
--keep_checkpoint_every_n_hour=6.0
fi
# get error mapping
sample_error_file="${sub_model_dir}/train.video_id.error"
if [ ! -f $sample_error_file ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-sample-error.py \
--output_file="$sample_error_file" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--deep_chain_layers=2 \
--deep_chain_relu_cells=128 \
--moe_num_mixtures=4 \
--lstm_layers=1 \
--lstm_cells="1024,128" \
--rnn_swap_memory=True \
--batch_size=128
fi
# generate resample freq file
output_freq_file="${sub_model_dir}/train.video_id.next_freq"
if [ ! -f $output_freq_file ]; then
python training_utils/reweight_sample_freq.py \
--clip_weight=5.0 \
--video_id_file="$vocab_file" \
--input_freq_file="$last_freq_file" \
--input_error_file="${sub_model_dir}/train.video_id.error" \
--output_freq_file="$output_freq_file"
fi
last_freq_file="${sub_model_dir}/train.video_id.next_freq"
echo "${model_name}/sub_model_$i" >> ${MODEL_DIR}/ensemble.conf
done
elif [[ $model_type =~ ^ensemble ]]; then
for i in {1..4}; do
sub_model_dir="${MODEL_DIR}/sub_model_${i}"
# inference-pre-ensemble
for part in test ensemble_validate ensemble_train; do
output_dir="/Youtube-8M/model_predictions_local/${part}/${model_name}/sub_model_${i}"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-pre-ensemble.py \
--output_dir="$output_dir" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--deep_chain_layers=2 \
--deep_chain_relu_cells=128 \
--moe_num_mixtures=4 \
--lstm_layers=1 \
--lstm_cells="1024,128" \
--rnn_swap_memory=True \
--batch_size=64 \
--file_size=4096
fi
done
done
# on ensemble server
cd ../youtube-8m-ensemble
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/train-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/eval-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/preensemble-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
fi
================================================
FILE: youtube-8m-wangheda/boosting_scripts/distillation-lstmparalleloutput-boosting.sh
================================================
#!/bin/bash
DEFAULT_GPU_ID=1
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
# base_model or sub_model_1 or sub_model_2 or so on
model_type="$1"
model_name="distillation_lstmparalleloutput_boosting"
MODEL="LstmParallelFinaloutputModel"
MODEL_DIR="../model/${model_name}"
vocab_file="resources/train.video_id.vocab"
default_freq_file="resources/train.video_id.freq"
if [ ! -f $vocab_file ]; then
cd resources
wget http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv
echo "OOV" > train.video_id.vocab
cat train_labels.csv | cut -d ',' -f 1 >> train.video_id.vocab
cd ..
fi
vocab_checksum=$(md5sum $vocab_file | cut -d ' ' -f 1)
if [ "$vocab_checksum" == "b74b8f2592cad5dd21bf614d1438db98" ]; then
echo $vocab_file is valid
else
echo $vocab_file is corrupted
exit 1
fi
if [ ! -f $default_freq_file ]; then
cat $vocab_file | awk '{print 1}' > $default_freq_file
fi
base_model_dir="${MODEL_DIR}/base_model"
if [ $model_type == "base_model" ]; then
# base model
rm ${MODEL_DIR}/ensemble.conf
mkdir -p $base_model_dir
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="$base_model_dir" \
--train_data_pattern="/Youtube-8M/distillation/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_type=1 \
--distillation_percent=0.5 \
--reweight=True \
--sample_vocab_file="$vocab_file" \
--sample_freq_file="$default_freq_file" \
--model=$MODEL \
--lstm_cells="1024,128" \
--moe_num_mixtures=8 \
--rnn_swap_memory=True \
--base_learning_rate=0.0008 \
--num_readers=2 \
--num_epochs=2 \
--batch_size=128 \
--keep_checkpoint_every_n_hour=72.0
elif [[ $model_type =~ ^sub_model ]]; then
last_freq_file=$default_freq_file
# sub model
for i in {1..3}; do
sub_model_dir="${MODEL_DIR}/sub_model_${i}"
if [ ! -d $sub_model_dir ]; then
cp -r $base_model_dir $sub_model_dir
echo "training model #$i, reweighting with $last_freq_file"
# train N models with re-weighted samples
CUDA_VISIBLE_DEVICES="$GPU_ID" python train-with-rebuild.py \
--train_dir="$sub_model_dir" \
--train_data_pattern="/Youtube-8M/distillation/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_type=1 \
--distillation_percent=0.5 \
--reweight=True \
--sample_vocab_file="resources/train.video_id.vocab" \
--sample_freq_file="$last_freq_file" \
--model=$MODEL \
--lstm_cells="1024,128" \
--moe_num_mixtures=8 \
--rnn_swap_memory=True \
--base_learning_rate=0.0008 \
--num_readers=2 \
--num_epochs=2 \
--batch_size=128 \
--keep_checkpoint_every_n_hour=72.0
fi
# get error mapping
sample_error_file="${sub_model_dir}/train.video_id.error"
if [ ! -f $sample_error_file ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-sample-error.py \
--output_file="$sample_error_file" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--lstm_cells="1024,128" \
--moe_num_mixtures=8 \
--rnn_swap_memory=True \
--batch_size=128
fi
# generate resample freq file
output_freq_file="${sub_model_dir}/train.video_id.next_freq"
if [ ! -f $output_freq_file ]; then
python training_utils/reweight_sample_freq.py \
--clip_weight=5.0 \
--video_id_file="$vocab_file" \
--input_freq_file="$last_freq_file" \
--input_error_file="${sub_model_dir}/train.video_id.error" \
--output_freq_file="$output_freq_file"
fi
last_freq_file="${sub_model_dir}/train.video_id.next_freq"
echo "${model_name}/sub_model_$i" >> ${MODEL_DIR}/ensemble.conf
done
elif [[ $model_type =~ ^ensemble ]]; then
for i in {1..4}; do
sub_model_dir="${MODEL_DIR}/sub_model_${i}"
# inference-pre-ensemble
for part in test ensemble_validate ensemble_train; do
output_dir="/Youtube-8M/model_predictions_local/${part}/${model_name}/sub_model_${i}"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-pre-ensemble.py \
--output_dir="$output_dir" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--lstm_cells="1024,128" \
--moe_num_mixtures=8 \
--rnn_swap_memory=True \
--batch_size=64 \
--file_size=4096
fi
done
done
# on ensemble server
cd ../youtube-8m-ensemble
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/train-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/eval-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/preensemble-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
fi
================================================
FILE: youtube-8m-wangheda/boosting_scripts/distillation-multilstm-dcc-boosting.sh
================================================
#!/bin/bash
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
# base_model or sub_model_1 or sub_model_2 or so on
model_type="$1"
model_name="distillation_multilstm_dcc_boosting"
MODEL_DIR="../model/${model_name}"
vocab_file="resources/train.video_id.vocab"
default_freq_file="resources/train.video_id.freq"
if [ ! -f $vocab_file ]; then
cd resources
wget http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv
echo "OOV" > train.video_id.vocab
cat train_labels.csv | cut -d ',' -f 1 >> train.video_id.vocab
cd ..
fi
vocab_checksum=$(md5sum $vocab_file | cut -d ' ' -f 1)
if [ "$vocab_checksum" == "b74b8f2592cad5dd21bf614d1438db98" ]; then
echo $vocab_file is valid
else
echo $vocab_file is corrupted
exit 1
fi
if [ ! -f $default_freq_file ]; then
cat $vocab_file | awk '{print 1}' > $default_freq_file
fi
base_model_dir="${MODEL_DIR}/base_model"
if [ $model_type == "base_model" ]; then
# base model
rm ${MODEL_DIR}/ensemble.conf
mkdir -p $base_model_dir
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="$base_model_dir" \
--train_data_pattern="/Youtube-8M/distillation/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_type=1 \
--distillation_percent=0.5 \
--reweight=True \
--sample_vocab_file="$vocab_file" \
--sample_freq_file="$default_freq_file" \
--model=LstmMemoryDeepChainModel \
--lstm_layers=2 \
--lstm_cells=1024 \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=1 \
--rnn_swap_memory=True \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_loss_percent=0.2 \
--support_type="label" \
--num_supports=4716 \
--base_learning_rate=0.0005 \
--num_readers=2 \
--num_epochs=2 \
--batch_size=128 \
--keep_checkpoint_every_n_hour=72.0
elif [[ $model_type =~ ^sub_model ]]; then
last_freq_file=$default_freq_file
# sub model
for i in {1..3}; do
sub_model_dir="${MODEL_DIR}/sub_model_${i}"
if [ ! -d $sub_model_dir ]; then
cp -r $base_model_dir $sub_model_dir
echo "training model #$i, reweighting with $last_freq_file"
# train N models with re-weighted samples
CUDA_VISIBLE_DEVICES="$GPU_ID" python train-with-rebuild.py \
--train_dir="$sub_model_dir" \
--train_data_pattern="/Youtube-8M/distillation/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_type=1 \
--distillation_percent=0.5 \
--reweight=True \
--sample_vocab_file="resources/train.video_id.vocab" \
--sample_freq_file="$last_freq_file" \
--model=LstmMemoryDeepChainModel \
--lstm_layers=2 \
--lstm_cells=1024 \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=1 \
--rnn_swap_memory=True \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_type="label" \
--support_loss_percent=0.2 \
--num_supports=4716 \
--base_learning_rate=0.0005 \
--num_readers=2 \
--num_epochs=2 \
--batch_size=128 \
--keep_checkpoint_every_n_hour=72.0
fi
# get error mapping
sample_error_file="${sub_model_dir}/train.video_id.error"
if [ ! -f $sample_error_file ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-sample-error.py \
--output_file="$sample_error_file" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=LstmMemoryDeepChainModel \
--lstm_layers=2 \
--lstm_cells=1024 \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=1 \
--rnn_swap_memory=True \
--batch_size=128
fi
# generate resample freq file
output_freq_file="${sub_model_dir}/train.video_id.next_freq"
if [ ! -f $output_freq_file ]; then
python training_utils/reweight_sample_freq.py \
--clip_weight=5.0 \
--video_id_file="$vocab_file" \
--input_freq_file="$last_freq_file" \
--input_error_file="${sub_model_dir}/train.video_id.error" \
--output_freq_file="$output_freq_file"
fi
last_freq_file="${sub_model_dir}/train.video_id.next_freq"
echo "${model_name}/sub_model_$i" >> ${MODEL_DIR}/ensemble.conf
done
elif [[ $model_type =~ ^ensemble ]]; then
for i in {1..3}; do
sub_model_dir="${MODEL_DIR}/sub_model_${i}"
# inference-pre-ensemble
for part in test ensemble_validate ensemble_train; do
output_dir="/Youtube-8M/model_predictions/${part}/${model_name}/sub_model_${i}"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-pre-ensemble.py \
--output_dir="$output_dir" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=LstmMemoryDeepChainModel \
--lstm_layers=2 \
--lstm_cells=1024 \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=1 \
--rnn_swap_memory=True \
--batch_size=64 \
--file_size=4096
fi
done
done
# on ensemble server
cd ../youtube-8m-ensemble
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/train-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/eval-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/preensemble-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
fi
================================================
FILE: youtube-8m-wangheda/boosting_scripts/distillation-multiscale-cnn-lstm-boosting.sh
================================================
#!/bin/bash
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
# base_model or sub_model_1 or sub_model_2 or so on
model_type="$1"
model_name="distillation_multiscale_cnnlstm_boosting"
MODEL="MultiscaleCnnLstmModel"
MODEL_DIR="../model/${model_name}"
vocab_file="resources/train.video_id.vocab"
default_freq_file="resources/train.video_id.freq"
if [ ! -f $vocab_file ]; then
cd resources
wget http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv
echo "OOV" > train.video_id.vocab
cat train_labels.csv | cut -d ',' -f 1 >> train.video_id.vocab
cd ..
fi
vocab_checksum=$(md5sum $vocab_file | cut -d ' ' -f 1)
if [ "$vocab_checksum" == "b74b8f2592cad5dd21bf614d1438db98" ]; then
echo $vocab_file is valid
else
echo $vocab_file is corrupted
exit 1
fi
if [ ! -f $default_freq_file ]; then
cat $vocab_file | awk '{print 1}' > $default_freq_file
fi
base_model_dir="${MODEL_DIR}/base_model"
if [ $model_type == "base_model" ]; then
# base model
rm ${MODEL_DIR}/ensemble.conf
mkdir -p $base_model_dir
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="$base_model_dir" \
--train_data_pattern="/Youtube-8M/distillation/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_type=1 \
--distillation_percent=0.5 \
--reweight=True \
--sample_vocab_file="$vocab_file" \
--sample_freq_file="$default_freq_file" \
--model=$MODEL \
--multiscale_cnn_lstm_layers=4 \
--moe_num_mixtures=4 \
--is_training=True \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_loss_percent=1.0 \
--support_type="label,label,label,label" \
--base_learning_rate=0.001 \
--num_readers=4 \
--num_epochs=3 \
--batch_size=128 \
--keep_checkpoint_every_n_hour=72.0
elif [[ $model_type =~ ^sub_model ]]; then
last_freq_file=$default_freq_file
# sub model
for i in {1..4}; do
sub_model_dir="${MODEL_DIR}/sub_model_${i}"
if [ ! -d $sub_model_dir ]; then
cp -r $base_model_dir $sub_model_dir
echo "training model #$i, reweighting with $last_freq_file"
# train N models with re-weighted samples
CUDA_VISIBLE_DEVICES="$GPU_ID" python train-with-rebuild.py \
--train_dir="$sub_model_dir" \
--train_data_pattern="/Youtube-8M/distillation/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_type=1 \
--distillation_percent=0.5 \
--reweight=True \
--sample_vocab_file="resources/train.video_id.vocab" \
--sample_freq_file="$last_freq_file" \
--model=$MODEL \
--multiscale_cnn_lstm_layers=4 \
--moe_num_mixtures=4 \
--is_training=True \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_loss_percent=1.0 \
--support_type="label,label,label,label" \
--base_learning_rate=0.001 \
--num_readers=4 \
--num_epochs=1 \
--batch_size=128 \
--keep_checkpoint_every_n_hour=72.0
fi
# get error mapping
sample_error_file="${sub_model_dir}/train.video_id.error"
if [ ! -f $sample_error_file ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-sample-error.py \
--output_file="$sample_error_file" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--multiscale_cnn_lstm_layers=4 \
--moe_num_mixtures=4 \
--batch_size=128
fi
# generate resample freq file
output_freq_file="${sub_model_dir}/train.video_id.next_freq"
if [ ! -f $output_freq_file ]; then
python training_utils/reweight_sample_freq.py \
--clip_weight=5.0 \
--video_id_file="$vocab_file" \
--input_freq_file="$last_freq_file" \
--input_error_file="${sub_model_dir}/train.video_id.error" \
--output_freq_file="$output_freq_file"
fi
last_freq_file="${sub_model_dir}/train.video_id.next_freq"
echo "${model_name}/sub_model_$i" >> ${MODEL_DIR}/ensemble.conf
done
elif [[ $model_type =~ ^ensemble ]]; then
for i in {1..4}; do
sub_model_dir="${MODEL_DIR}/sub_model_${i}"
# inference-pre-ensemble
for part in test ensemble_validate ensemble_train; do
output_dir="/Youtube-8M/model_predictions/${part}/${model_name}/sub_model_${i}"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-pre-ensemble.py \
--output_dir="$output_dir" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--multiscale_cnn_lstm_layers=4 \
--moe_num_mixtures=4 \
--batch_size=64 \
--file_size=4096
fi
done
done
# on ensemble server
cd ../youtube-8m-ensemble
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/train-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/eval-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/preensemble-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
fi
================================================
FILE: youtube-8m-wangheda/boosting_scripts/distillation-positional-lstmattention8max-boosting.sh
================================================
#!/bin/bash
DEFAULT_GPU_ID=1
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
# base_model or sub_model_1 or sub_model_2 or so on
model_type="$1"
model_name="distillation_positional_lstmattention8max_boosting"
MODEL="LstmPositionalAttentionMaxPoolingModel"
MODEL_DIR="../model/${model_name}"
vocab_file="resources/train.video_id.vocab"
default_freq_file="resources/train.video_id.freq"
if [ ! -f $vocab_file ]; then
cd resources
wget http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv
echo "OOV" > train.video_id.vocab
cat train_labels.csv | cut -d ',' -f 1 >> train.video_id.vocab
cd ..
fi
vocab_checksum=$(md5sum $vocab_file | cut -d ' ' -f 1)
if [ "$vocab_checksum" == "b74b8f2592cad5dd21bf614d1438db98" ]; then
echo $vocab_file is valid
else
echo $vocab_file is corrupted
exit 1
fi
if [ ! -f $default_freq_file ]; then
cat $vocab_file | awk '{print 1}' > $default_freq_file
fi
base_model_dir="${MODEL_DIR}/base_model"
if [ $model_type == "base_model" ]; then
# base model
rm ${MODEL_DIR}/ensemble.conf
mkdir -p $base_model_dir
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="$base_model_dir" \
--train_data_pattern="/Youtube-8M/distillation/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_type=1 \
--distillation_percent=0.5 \
--reweight=True \
--sample_vocab_file="$vocab_file" \
--sample_freq_file="$default_freq_file" \
--model=$MODEL \
--moe_num_mixtures=8 \
--lstm_attentions=8 \
--positional_embedding_size=32 \
--rnn_swap_memory=True \
--base_learning_rate=0.001 \
--num_readers=4 \
--num_epochs=3 \
--batch_size=96 \
--keep_checkpoint_every_n_hour=6.0
elif [[ $model_type =~ ^sub_model ]]; then
last_freq_file=$default_freq_file
# sub model
for i in {1..4}; do
sub_model_dir="${MODEL_DIR}/sub_model_${i}"
if [ ! -d $sub_model_dir ]; then
cp -r $base_model_dir $sub_model_dir
echo "training model #$i, reweighting with $last_freq_file"
# train N models with re-weighted samples
CUDA_VISIBLE_DEVICES="$GPU_ID" python train-with-rebuild.py \
--train_dir="$sub_model_dir" \
--train_data_pattern="/Youtube-8M/distillation/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_type=1 \
--distillation_percent=0.5 \
--reweight=True \
--sample_vocab_file="resources/train.video_id.vocab" \
--sample_freq_file="$last_freq_file" \
--model=$MODEL \
--moe_num_mixtures=8 \
--lstm_attentions=8 \
--positional_embedding_size=32 \
--rnn_swap_memory=True \
--base_learning_rate=0.001 \
--num_readers=2 \
--num_epochs=2 \
--batch_size=96 \
--keep_checkpoint_every_n_hour=6.0
fi
# get error mapping
sample_error_file="${sub_model_dir}/train.video_id.error"
if [ ! -f $sample_error_file ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-sample-error.py \
--output_file="$sample_error_file" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--moe_num_mixtures=8 \
--lstm_attentions=8 \
--positional_embedding_size=32 \
--rnn_swap_memory=True \
--batch_size=128
fi
# generate resample freq file
output_freq_file="${sub_model_dir}/train.video_id.next_freq"
if [ ! -f $output_freq_file ]; then
python training_utils/reweight_sample_freq.py \
--clip_weight=5.0 \
--video_id_file="$vocab_file" \
--input_freq_file="$last_freq_file" \
--input_error_file="${sub_model_dir}/train.video_id.error" \
--output_freq_file="$output_freq_file"
fi
last_freq_file="${sub_model_dir}/train.video_id.next_freq"
echo "${model_name}/sub_model_$i" >> ${MODEL_DIR}/ensemble.conf
done
elif [[ $model_type =~ ^ensemble ]]; then
for i in {1..4}; do
sub_model_dir="${MODEL_DIR}/sub_model_${i}"
# inference-pre-ensemble
for part in test ensemble_validate ensemble_train; do
output_dir="/Youtube-8M/model_predictions/${part}/${model_name}/sub_model_${i}"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-pre-ensemble.py \
--output_dir="$output_dir" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--moe_num_mixtures=8 \
--lstm_attentions=8 \
--positional_embedding_size=32 \
--rnn_swap_memory=True \
--batch_size=64 \
--file_size=4096
fi
done
done
# on ensemble server
cd ../youtube-8m-ensemble
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/train-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/eval-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/preensemble-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
fi
================================================
FILE: youtube-8m-wangheda/boosting_scripts/distillation-video-dcc-boosting.sh
================================================
#!/bin/bash
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
model_name="distillation_video_dcc_boosting"
MODEL="DeepCombineChainModel"
MODEL_DIR="../model/${model_name}"
vocab_file="resources/train.video_id.vocab"
default_freq_file="resources/train.video_id.freq"
if [ ! -f $vocab_file ]; then
cd resources
wget http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv
echo "OOV" > train.video_id.vocab
cat train_labels.csv | cut -d ',' -f 1 >> train.video_id.vocab
cd ..
fi
vocab_checksum=$(md5sum $vocab_file | cut -d ' ' -f 1)
if [ "$vocab_checksum" == "b74b8f2592cad5dd21bf614d1438db98" ]; then
echo $vocab_file is valid
else
echo $vocab_file is corrupted
exit 1
fi
if [ ! -f $default_freq_file ]; then
cat $vocab_file | awk '{print 1}' > $default_freq_file
fi
rm ${MODEL_DIR}/ensemble.conf
# base model
last_freq_file=$default_freq_file
# sub model
for i in {1..4}; do
sub_model_dir="${MODEL_DIR}/sub_model_${i}"
if [ ! -d $sub_model_dir ]; then
echo "training model #$i, reweighting with $last_freq_file"
# train N models with re-weighted samples
CUDA_VISIBLE_DEVICES="$GPU_ID" python train-with-rebuild.py \
--train_dir="$sub_model_dir" \
--train_data_pattern="/Youtube-8M/distillation/video/train/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_type=1 \
--distillation_percent=0.5 \
--reweight=True \
--sample_vocab_file="resources/train.video_id.vocab" \
--sample_freq_file="$last_freq_file" \
--model=$MODEL \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=4 \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label,label" \
--support_loss_percent=0.05 \
--base_learning_rate=0.008 \
--num_readers=4 \
--num_epochs=5 \
--batch_size=512 \
--keep_checkpoint_interval=6 \
--keep_checkpoint_every_n_hour=72.0
fi
# get error mapping
sample_error_file="${sub_model_dir}/train.video_id.error"
if [ ! -f $sample_error_file ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-sample-error.py \
--output_file="$sample_error_file" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/video/train/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=4 \
--batch_size=128
fi
# generate resample freq file
output_freq_file="${sub_model_dir}/train.video_id.next_freq"
if [ ! -f $output_freq_file ]; then
python training_utils/reweight_sample_freq.py \
--clip_weight=8.0 \
--video_id_file="$vocab_file" \
--input_freq_file="$last_freq_file" \
--input_error_file="${sub_model_dir}/train.video_id.error" \
--output_freq_file="$output_freq_file"
fi
# inference-pre-ensemble
for part in test ensemble_validate ensemble_train; do
output_dir="/Youtube-8M/model_predictions_local/${part}/${model_name}/sub_model_${i}"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-pre-ensemble.py \
--output_dir="$output_dir" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=4 \
--batch_size=64 \
--file_size=4096
fi
done
last_freq_file="${sub_model_dir}/train.video_id.next_freq"
echo "${model_name}/sub_model_$i" >> ${MODEL_DIR}/ensemble.conf
done
# on ensemble server
cd ../youtube-8m-ensemble
CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/train-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf _local
CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/eval-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf _local
for part in ensemble_train ensemble_validate test; do
CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/auto-preensemble-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf $part _local
done
================================================
FILE: youtube-8m-wangheda/boosting_scripts/lstmattention8max-boosting-weightclip.sh
================================================
#!/bin/bash
DEFAULT_GPU_ID=1
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
# base_model or sub_model_1 or sub_model_2 or so on
model_type="$1"
model_name="lstmattention8max_boosting"
MODEL_DIR="../model/${model_name}"
vocab_file="resources/train.video_id.vocab"
default_freq_file="resources/train.video_id.freq"
if [ ! -f $vocab_file ]; then
cd resources
wget http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv
echo "OOV" > train.video_id.vocab
cat train_labels.csv | cut -d ',' -f 1 >> train.video_id.vocab
cd ..
fi
vocab_checksum=$(md5sum $vocab_file | cut -d ' ' -f 1)
if [ "$vocab_checksum" == "b74b8f2592cad5dd21bf614d1438db98" ]; then
echo $vocab_file is valid
else
echo $vocab_file is corrupted
exit 1
fi
if [ ! -f $default_freq_file ]; then
cat $vocab_file | awk '{print 1}' > $default_freq_file
fi
base_model_dir="${MODEL_DIR}/base_model"
if [ $model_type == "base_model" ]; then
# base model
rm ${MODEL_DIR}/ensemble.conf
mkdir -p $base_model_dir
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="$base_model_dir" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--reweight=True \
--sample_vocab_file="$vocab_file" \
--sample_freq_file="$default_freq_file" \
--model=LstmAttentionMaxPoolingModel \
--moe_num_mixtures=8 \
--lstm_attentions=8 \
--rnn_swap_memory=True \
--base_learning_rate=0.001 \
--num_readers=2 \
--num_epochs=4 \
--batch_size=128 \
--keep_checkpoint_every_n_hour=0.5
elif [[ $model_type =~ ^sub_model ]]; then
last_freq_file=$default_freq_file
# sub model
for i in {1..8}; do
sub_model_dir="${MODEL_DIR}/sub_model_${i}"
if [ ! -d $sub_model_dir ]; then
cp -r $base_model_dir $sub_model_dir
echo "training model #$i, reweighting with $last_freq_file"
# train N models with re-weighted samples
CUDA_VISIBLE_DEVICES="$GPU_ID" python train-with-rebuild.py \
--train_dir="$sub_model_dir" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--reweight=True \
--sample_vocab_file="resources/train.video_id.vocab" \
--sample_freq_file="$last_freq_file" \
--model=LstmAttentionMaxPoolingModel \
--moe_num_mixtures=8 \
--lstm_attentions=8 \
--rnn_swap_memory=True \
--base_learning_rate=0.001 \
--num_readers=4 \
--num_epochs=1 \
--batch_size=128 \
--keep_checkpoint_every_n_hour=72.0
fi
# get error mapping
sample_error_file="${sub_model_dir}/train.video_id.error"
if [ ! -f $sample_error_file ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-sample-error.py \
--output_file="$sample_error_file" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=LstmAttentionMaxPoolingModel \
--moe_num_mixtures=8 \
--lstm_attentions=8 \
--rnn_swap_memory=True \
--batch_size=128
fi
# generate resample freq file
output_freq_file="${sub_model_dir}/train.video_id.next_freq"
if [ ! -f $output_freq_file ]; then
python training_utils/reweight_sample_freq.py \
--clip_weight=5.0 \
--video_id_file="$vocab_file" \
--input_freq_file="$last_freq_file" \
--input_error_file="${sub_model_dir}/train.video_id.error" \
--output_freq_file="$output_freq_file"
fi
last_freq_file="${sub_model_dir}/train.video_id.next_freq"
echo "${model_name}/sub_model_$i" >> ${MODEL_DIR}/ensemble.conf
done
elif [[ $model_type =~ ^ensemble ]]; then
for i in {1..2}; do
sub_model_dir="${MODEL_DIR}/sub_model_${i}"
# inference-pre-ensemble
for part in test ensemble_validate ensemble_train; do
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/${model_name}/sub_model_${i}" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=LstmAttentionMaxPoolingModel \
--moe_num_mixtures=8 \
--lstm_attentions=8 \
--rnn_swap_memory=True \
--batch_size=32 \
--file_size=4096
done
done
# on ensemble server
#cd ../youtube-8m-ensemble
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/train-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/eval-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/infer-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
fi
================================================
FILE: youtube-8m-wangheda/boosting_scripts/lstmparalleloutput-boosting-weightclip.sh
================================================
#!/bin/bash
DEFAULT_GPU_ID=1
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
# base_model or sub_model_1 or sub_model_2 or so on
model_type="$1"
model_name="lstmparalleloutput_boosting_weightclip"
MODEL_DIR="../model/${model_name}"
vocab_file="resources/train.video_id.vocab"
default_freq_file="resources/train.video_id.freq"
if [ ! -f $vocab_file ]; then
cd resources
wget http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv
echo "OOV" > train.video_id.vocab
cat train_labels.csv | cut -d ',' -f 1 >> train.video_id.vocab
cd ..
fi
vocab_checksum=$(md5sum $vocab_file | cut -d ' ' -f 1)
if [ "$vocab_checksum" == "b74b8f2592cad5dd21bf614d1438db98" ]; then
echo $vocab_file is valid
else
echo $vocab_file is corrupted
exit 1
fi
if [ ! -f $default_freq_file ]; then
cat $vocab_file | awk '{print 1}' > $default_freq_file
fi
base_model_dir="${MODEL_DIR}/base_model"
if [ $model_type == "base_model" ]; then
# base model
rm ${MODEL_DIR}/ensemble.conf
mkdir -p $base_model_dir
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="$base_model_dir" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--reweight=True \
--sample_vocab_file="$vocab_file" \
--sample_freq_file="$default_freq_file" \
--model=LstmParallelFinaloutputModel \
--lstm_cells="1024,128" \
--moe_num_mixtures=8 \
--rnn_swap_memory=True \
--base_learning_rate=0.0008 \
--num_readers=2 \
--num_epochs=3 \
--batch_size=128 \
--keep_checkpoint_every_n_hour=2.0
elif [[ $model_type =~ ^sub_model ]]; then
last_freq_file=$default_freq_file
# sub model
for i in {1..8}; do
sub_model_dir="${MODEL_DIR}/sub_model_${i}"
if [ ! -d $sub_model_dir ]; then
cp -r $base_model_dir $sub_model_dir
echo "training model #$i, reweighting with $last_freq_file"
# train N models with re-weighted samples
CUDA_VISIBLE_DEVICES="$GPU_ID" python train-with-rebuild.py \
--train_dir="$sub_model_dir" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--reweight=True \
--sample_vocab_file="resources/train.video_id.vocab" \
--sample_freq_file="$last_freq_file" \
--model=LstmParallelFinaloutputModel \
--lstm_cells="1024,128" \
--moe_num_mixtures=8 \
--rnn_swap_memory=True \
--base_learning_rate=0.0008 \
--num_readers=2 \
--num_epochs=1 \
--batch_size=128 \
--keep_checkpoint_every_n_hour=72.0
fi
# get error mapping
sample_error_file="${sub_model_dir}/train.video_id.error"
if [ ! -f $sample_error_file ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-sample-error.py \
--output_file="$sample_error_file" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=LstmParallelFinaloutputModel \
--lstm_cells="1024,128" \
--moe_num_mixtures=8 \
--rnn_swap_memory=True \
--batch_size=128
fi
# generate resample freq file
output_freq_file="${sub_model_dir}/train.video_id.next_freq"
if [ ! -f $output_freq_file ]; then
python training_utils/reweight_sample_freq.py \
--clip_weight=5.0 \
--video_id_file="$vocab_file" \
--input_freq_file="$last_freq_file" \
--input_error_file="${sub_model_dir}/train.video_id.error" \
--output_freq_file="$output_freq_file"
fi
last_freq_file="${sub_model_dir}/train.video_id.next_freq"
echo "${model_name}/sub_model_$i" >> ${MODEL_DIR}/ensemble.conf
done
elif [[ $model_type =~ ^ensemble ]]; then
for i in {1..8}; do
sub_model_dir="${MODEL_DIR}/sub_model_${i}"
# inference-pre-ensemble
for part in test ensemble_validate ensemble_train; do
output_dir="/Youtube-8M/model_predictions/${part}/${model_name}/sub_model_${i}"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-pre-ensemble.py \
--output_dir="$output_dir" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=LstmParallelFinaloutputModel \
--lstm_cells="1024,128" \
--moe_num_mixtures=8 \
--rnn_swap_memory=True \
--batch_size=64 \
--file_size=4096
fi
done
done
# on ensemble server
#cd ../youtube-8m-ensemble
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/train-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/eval-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/infer-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
fi
================================================
FILE: youtube-8m-wangheda/boosting_scripts/video-deep-combine-chain-boosting-discardhopeless.sh
================================================
#!/bin/bash
# base_model or sub_model_1 or sub_model_2 or so on
#model_type="$1"
model_name="video_dcc_boosting_discardhopeless"
MODEL_DIR="../model/${model_name}"
vocab_file="resources/train.video_id.vocab"
default_freq_file="resources/train.video_id.freq"
rm ${MODEL_DIR}/ensemble.conf
if [ ! -f $vocab_file ]; then
cd resources
wget http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv
echo "OOV" > train.video_id.vocab
cat train_labels.csv | cut -d ',' -f 1 >> train.video_id.vocab
cd ..
fi
vocab_checksum=$(md5sum $vocab_file | cut -d ' ' -f 1)
if [ "$vocab_checksum" == "b74b8f2592cad5dd21bf614d1438db98" ]; then
echo $vocab_file is valid
else
echo $vocab_file is corrupted
exit 1
fi
if [ ! -f $default_freq_file ]; then
cat $vocab_file | awk '{print 1}' > $default_freq_file
fi
base_model_dir="${MODEL_DIR}/base_model"
# base model (4 epochs)
if [ ! -d $base_model_dir ]; then
mkdir -p $base_model_dir
for j in {1..2}; do
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="$base_model_dir" \
--train_data_pattern="/Youtube-8M/data/video/train/train*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=DeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=4 \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label,label" \
--num_supports=18864 \
--support_loss_percent=0.05 \
--reweight=True \
--sample_vocab_file="$vocab_file" \
--sample_freq_file="$default_freq_file" \
--keep_checkpoint_every_n_hour=8.0 \
--keep_checkpoint_interval=6 \
--base_learning_rate=0.01 \
--data_augmenter=NoiseAugmenter \
--input_noise_level=0.2 \
--num_readers=2 \
--num_epochs=2 \
--batch_size=1024
done
fi
last_freq_file=$default_freq_file
for i in {1..8}; do
sub_model_dir="${MODEL_DIR}/sub_model_${i}"
if [ ! -d $sub_model_dir ]; then
cp -r $base_model_dir $sub_model_dir
echo "training model #$i, reweighting with $last_freq_file"
# train N models with re-weighted samples
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="$sub_model_dir" \
--train_data_pattern="/Youtube-8M/data/video/train/train*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=DeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=4 \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label,label" \
--num_supports=18864 \
--support_loss_percent=0.05 \
--reweight=True \
--sample_vocab_file="$vocab_file" \
--sample_freq_file="$last_freq_file" \
--keep_checkpoint_every_n_hour=8.0 \
--base_learning_rate=0.01 \
--data_augmenter=NoiseAugmenter \
--input_noise_level=0.2 \
--num_readers=2 \
--num_epochs=2 \
--batch_size=1024
fi
# inference-pre-ensemble
for part in test ensemble_validate ensemble_train; do
output_dir="/Youtube-8M/model_predictions/${part}/${model_name}/sub_model_$i"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="${output_dir}" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=DeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=4 \
--batch_size=128 \
--file_size=4096
fi
done
# get error mapping
output_error_file="${sub_model_dir}/train.video_id.error"
if [ ! -f $output_error_file ]; then
echo "generating error mapping to $output_error_file"
CUDA_VISIBLE_DEVICES=0 python inference-sample-error.py \
--output_file="${output_error_file}" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/video/train/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=DeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=4 \
--batch_size=1024
fi
# generate resample freq file
output_freq_file="${sub_model_dir}/train.video_id.next_freq"
if [ ! -f $output_freq_file ]; then
echo "generating reweight freq to $output_freq_file"
python training_utils/reweight_sample_freq.py \
--discard_weight=20.0 \
--video_id_file="$vocab_file" \
--input_freq_file="$last_freq_file" \
--input_error_file="${sub_model_dir}/train.video_id.error" \
--output_freq_file="${sub_model_dir}/train.video_id.next_freq"
fi
last_freq_file="${sub_model_dir}/train.video_id.next_freq"
echo "${model_name}/sub_model_$i" >> ${MODEL_DIR}/ensemble.conf
done
cd ../youtube-8m-ensemble
# partly ensemble
for i in 1 2 4; do
part_conf="${MODEL_DIR}/ensemble${i}.conf"
cat ${MODEL_DIR}/ensemble.conf | head -n $i > $part_conf
bash ensemble_scripts/train-matrix_model.sh ${model_name}/ensemble${i}_matrix_model $part_conf
bash ensemble_scripts/eval-matrix_model.sh ${model_name}/ensemble${i}_matrix_model $part_conf
done
# all ensemble
bash ensemble_scripts/train-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
bash ensemble_scripts/eval-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
#bash ensemble_scripts/infer-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
================================================
FILE: youtube-8m-wangheda/boosting_scripts/video-deep-combine-chain-boosting-weightclip.sh
================================================
#!/bin/bash
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
# base_model or sub_model_1 or sub_model_2 or so on
#model_type="$1"
model_name="video_dcc_boosting_weightclip"
MODEL_DIR="../model/${model_name}"
vocab_file="resources/train.video_id.vocab"
default_freq_file="resources/train.video_id.freq"
rm ${MODEL_DIR}/ensemble.conf
if [ ! -f $vocab_file ]; then
cd resources
wget http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv
echo "OOV" > train.video_id.vocab
cat train_labels.csv | cut -d ',' -f 1 >> train.video_id.vocab
cd ..
fi
vocab_checksum=$(md5sum $vocab_file | cut -d ' ' -f 1)
if [ "$vocab_checksum" == "b74b8f2592cad5dd21bf614d1438db98" ]; then
echo $vocab_file is valid
else
echo $vocab_file is corrupted
exit 1
fi
if [ ! -f $default_freq_file ]; then
cat $vocab_file | awk '{print 1}' > $default_freq_file
fi
base_model_dir="${MODEL_DIR}/base_model"
# base model (4 epochs)
if [ ! -d $base_model_dir ]; then
mkdir -p $base_model_dir
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="$base_model_dir" \
--train_data_pattern="/Youtube-8M/data/video/train/train*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=DeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=4 \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label,label" \
--num_supports=18864 \
--support_loss_percent=0.05 \
--reweight=True \
--sample_vocab_file="$vocab_file" \
--sample_freq_file="$default_freq_file" \
--keep_checkpoint_every_n_hour=8.0 \
--keep_checkpoint_interval=6 \
--base_learning_rate=0.01 \
--data_augmenter=NoiseAugmenter \
--input_noise_level=0.2 \
--num_readers=2 \
--num_epochs=4 \
--batch_size=1024
fi
last_freq_file=$default_freq_file
for i in {1..8}; do
sub_model_dir="${MODEL_DIR}/sub_model_${i}"
if [ ! -d $sub_model_dir ]; then
cp -r $base_model_dir $sub_model_dir
echo "training model #$i, reweighting with $last_freq_file"
# train N models with re-weighted samples
CUDA_VISIBLE_DEVICES="$GPU_ID" python train-with-rebuild.py \
--train_dir="$sub_model_dir" \
--train_data_pattern="/Youtube-8M/data/video/train/train*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=DeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=4 \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label,label" \
--num_supports=18864 \
--support_loss_percent=0.05 \
--reweight=True \
--sample_vocab_file="$vocab_file" \
--sample_freq_file="$last_freq_file" \
--keep_checkpoint_every_n_hour=8.0 \
--base_learning_rate=0.01 \
--data_augmenter=NoiseAugmenter \
--input_noise_level=0.2 \
--num_readers=2 \
--num_epochs=2 \
--batch_size=1024
fi
# inference-pre-ensemble
for part in test ensemble_validate ensemble_train; do
output_dir="/Youtube-8M/model_predictions/${part}/${model_name}/sub_model_$i"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-pre-ensemble.py \
--output_dir="${output_dir}" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=DeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=4 \
--batch_size=128 \
--file_size=4096
fi
done
# get error mapping
output_error_file="${sub_model_dir}/train.video_id.error"
if [ ! -f $output_error_file ]; then
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-sample-error.py \
--output_file="${output_error_file}" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/video/train/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=DeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=4 \
--batch_size=1024
fi
# generate resample freq file
output_freq_file="${sub_model_dir}/train.video_id.next_freq"
if [ ! -f $output_freq_file ]; then
python training_utils/reweight_sample_freq.py \
--clip_weight=5.0 \
--video_id_file="$vocab_file" \
--input_freq_file="$last_freq_file" \
--input_error_file="${sub_model_dir}/train.video_id.error" \
--output_freq_file="${sub_model_dir}/train.video_id.next_freq"
fi
last_freq_file="${sub_model_dir}/train.video_id.next_freq"
echo "${model_name}/sub_model_$i" >> ${MODEL_DIR}/ensemble.conf
done
cd ../youtube-8m-ensemble
# all ensemble
CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/train-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/eval-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/infer-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
================================================
FILE: youtube-8m-wangheda/boosting_scripts/video-deep-combine-chain-boosting.sh
================================================
#!/bin/bash
DEFAULT_GPU_ID=0
if [ -z ${CUDA_VISIBLE_DEVICES+x} ]; then
GPU_ID=$DEFAULT_GPU_ID
echo "set CUDA_VISIBLE_DEVICES to default('$GPU_ID')"
else
GPU_ID=$CUDA_VISIBLE_DEVICES
echo "set CUDA_VISIBLE_DEVICES to external('$GPU_ID')"
fi
# base_model or sub_model_1 or sub_model_2 or so on
#model_type="$1"
model_name="video_dcc_boosting"
MODEL_DIR="../model/${model_name}"
vocab_file="resources/train.video_id.vocab"
default_freq_file="resources/train.video_id.freq"
rm ${MODEL_DIR}/ensemble.conf
if [ ! -f $vocab_file ]; then
cd resources
wget http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv
echo "OOV" > train.video_id.vocab
cat train_labels.csv | cut -d ',' -f 1 >> train.video_id.vocab
cd ..
fi
vocab_checksum=$(md5sum $vocab_file | cut -d ' ' -f 1)
if [ "$vocab_checksum" == "b74b8f2592cad5dd21bf614d1438db98" ]; then
echo $vocab_file is valid
else
echo $vocab_file is corrupted
exit 1
fi
if [ ! -f $default_freq_file ]; then
cat $vocab_file | awk '{print 1}' > $default_freq_file
fi
base_model_dir="${MODEL_DIR}/base_model"
# base model (4 epochs)
mkdir -p $base_model_dir
for j in {1..2}; do
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="$base_model_dir" \
--train_data_pattern="/Youtube-8M/data/video/train/train*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=DeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=4 \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label,label" \
--num_supports=18864 \
--support_loss_percent=0.05 \
--reweight=True \
--sample_vocab_file="$vocab_file" \
--sample_freq_file="$default_freq_file" \
--keep_checkpoint_every_n_hour=8.0 \
--keep_checkpoint_interval=6 \
--base_learning_rate=0.01 \
--data_augmenter=NoiseAugmenter \
--input_noise_level=0.2 \
--num_readers=2 \
--num_epochs=2 \
--batch_size=1024
done
last_freq_file=$default_freq_file
for i in {1..8}; do
sub_model_dir="${MODEL_DIR}/sub_model_${i}"
cp -r $base_model_dir $sub_model_dir
echo "training model #$i, reweighting with $last_freq_file"
# train N models with re-weighted samples
CUDA_VISIBLE_DEVICES="$GPU_ID" python train.py \
--train_dir="$sub_model_dir" \
--train_data_pattern="/Youtube-8M/data/video/train/train*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=DeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=4 \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label,label" \
--num_supports=18864 \
--support_loss_percent=0.05 \
--reweight=True \
--sample_vocab_file="$vocab_file" \
--sample_freq_file="$last_freq_file" \
--keep_checkpoint_every_n_hour=8.0 \
--base_learning_rate=0.01 \
--data_augmenter=NoiseAugmenter \
--input_noise_level=0.2 \
--num_readers=2 \
--num_epochs=2 \
--batch_size=1024
# inference-pre-ensemble
for part in test ensemble_validate ensemble_train; do
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/${model_name}/sub_model_$i" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--file_size=4096
done
# get error mapping
CUDA_VISIBLE_DEVICES="$GPU_ID" python inference-sample-error.py \
--output_file="${sub_model_dir}/train.video_id.error" \
--train_dir="${sub_model_dir}" \
--input_data_pattern="/Youtube-8M/data/video/train/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=DeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=4 \
--batch_size=1024
# generate resample freq file
python training_utils/reweight_sample_freq.py \
--video_id_file="$vocab_file" \
--input_freq_file="$last_freq_file" \
--input_error_file="${sub_model_dir}/train.video_id.error" \
--output_freq_file="${sub_model_dir}/train.video_id.next_freq"
last_freq_file="${sub_model_dir}/train.video_id.next_freq"
echo "${model_name}/sub_model_$i" >> ${MODEL_DIR}/ensemble.conf
done
cd ../youtube-8m-ensemble
CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/train-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/eval-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
#CUDA_VISIBLE_DEVICES="$GPU_ID" bash ensemble_scripts/infer-matrix_model.sh ${model_name}/ensemble_matrix_model ${MODEL_DIR}/ensemble.conf
================================================
FILE: youtube-8m-wangheda/cascade_scripts/distillchain-v2-hybridchain.sh
================================================
#!/bin/bash
task=$1
model_name="hybridchain"
model_path="../model/${model_name}"
if [ $task == "train" ]; then
predictions_data_pattern="/Youtube-8M/model_predictions/train/distillation/ensemble_v2_matrix_model/*.tfrecord"
# train model 1: Video-DCC
sub_model_name="distillchain_v2_video_dcc"
sub_model_path="${model_path}/${sub_model_name}"
if [ ! -d $sub_model_path ]; then
mkdir -p $sub_model_path
echo predictions = "$predictions_data_pattern"
CUDA_VISIBLE_DEVICES=0 python train-with-predictions.py \
--train_dir="$sub_model_path" \
--train_data_pattern="/Youtube-8M/data/video/train/*.tfrecord" \
--predictions_data_pattern="$predictions_data_pattern" \
--distillation_features=False \
--distillation_as_input=True \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=DistillchainDeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_layers=3 \
--deep_chain_relu_cells=256 \
--data_augmenter=NoiseAugmenter \
--input_noise_level=0.1 \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_type="label,label,label" \
--support_loss_percent=0.05 \
--base_learning_rate=0.01 \
--keep_checkpoint_every_n_hour=5.0 \
--keep_checkpoint_interval=6 \
--num_epochs=2 \
--batch_size=1024
fi
# inference model 1
infer_data_path="/Youtube-8M/model_predictions/train/${model_name}/${sub_model_name}"
if [ ! -d $infer_data_path ]; then
echo predictions = "$predictions_data_pattern"
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble-with-predictions.py \
--output_dir="$infer_data_path" \
--train_dir="$sub_model_path" \
--input_data_pattern="/Youtube-8M/data/video/train/*.tfrecord" \
--distill_data_pattern="$predictions_data_pattern" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--distillation_features=False \
--distillation_as_input=True \
--model=DistillchainDeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_layers=3 \
--deep_chain_relu_cells=256 \
--batch_size=1024 \
--file_size=4096
fi
# next predictions pattern
predictions_data_pattern="${predictions_data_pattern},${infer_data_path}/*.tfrecord"
# train model 2: CNN-DCC
sub_model_name="distillchain_v2_cnn_dcc"
sub_model_path="${model_path}/${sub_model_name}"
if [ ! -d $sub_model_path ]; then
mkdir -p $sub_model_path
echo predictions = "$predictions_data_pattern"
CUDA_VISIBLE_DEVICES=0 python train-with-predictions.py \
--predictions_data_pattern="$predictions_data_pattern" \
--distillation_features=True \
--distillation_as_input=True \
--train_dir="$sub_model_path" \
--train_data_pattern="/Youtube-8M/data/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=DistillchainCnnDeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_layers=3 \
--deep_chain_relu_cells=256 \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_type="label,label,label" \
--support_loss_percent=0.05 \
--num_readers=4 \
--batch_size=128 \
--num_epochs=2 \
--keep_checkpoint_every_n_hours=20 \
--base_learning_rate=0.001
fi
# inference model 2
infer_data_path="/Youtube-8M/model_predictions/train/${model_name}/${sub_model_name}"
if [ ! -d $infer_data_path ]; then
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble-with-predictions.py \
--output_dir="$infer_data_path" \
--train_dir="$sub_model_path" \
--input_data_pattern="/Youtube-8M/data/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distill_data_pattern="$predictions_data_pattern" \
--distillation_features=True \
--distillation_as_input=True \
--model=DistillchainCnnDeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_layers=3 \
--deep_chain_relu_cells=256 \
--batch_size=128 \
--file_size=4096
fi
# next predictions pattern
predictions_data_pattern="${predictions_data_pattern},${infer_data_path}/*.tfrecord"
# train model 2: LSTM
sub_model_name="distillchain_v2_lstmparalleloutput"
sub_model_path="${model_path}/${sub_model_name}"
if [ ! -d $sub_model_path ]; then
mkdir -p $sub_model_path
echo predictions = "$predictions_data_pattern"
CUDA_VISIBLE_DEVICES=0 python train-with-predictions.py \
--predictions_data_pattern="$predictions_data_pattern" \
--distillation_features=True \
--distillation_as_input=True \
--train_dir="$sub_model_path" \
--train_data_pattern="/Youtube-8M/data/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=DistillchainLstmParallelFinaloutputModel \
--rnn_swap_memory=True \
--lstm_layers=1 \
--moe_num_mixtures=8 \
--lstm_cells="1024,128" \
--batch_size=128 \
--num_epochs=1 \
--keep_checkpoint_every_n_hours=20 \
--base_learning_rate=0.0008
fi
elif [ $task == "test-lstm" ]; then
sub_model_name="distillchain_v2_lstmparalleloutput"
sub_model_path="${model_path}/${sub_model_name}"
if [ -d $sub_model_path ]; then
for part in ensemble_train ensemble_validate test; do
infer_data_path="/Youtube-8M/model_predictions/${part}/${model_name}/${sub_model_name}"
if [ ! -d $infer_data_path ]; then
predictions_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_v2_matrix_model/*.tfrecord"
for prev_model in distillchain_v2_video_dcc distillchain_v2_cnn_dcc; do
predictions_data_pattern="${predictions_data_pattern},/Youtube-8M/model_predictions/${part}/${model_name}/${prev_model}/*.tfrecord"
done
echo predictions = "$predictions_data_pattern"
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble-with-predictions.py \
--output_dir="$infer_data_path" \
--train_dir="$sub_model_path" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distill_data_pattern="$predictions_data_pattern" \
--distillation_features=True \
--distillation_as_input=True \
--model=DistillchainLstmParallelFinaloutputModel \
--rnn_swap_memory=True \
--lstm_layers=1 \
--moe_num_mixtures=8 \
--lstm_cells="1024,128" \
--batch_size=128 \
--file_size=4096
fi
done
fi
elif [ $task == "test-cnn" ]; then
sub_model_name="distillchain_v2_cnn_dcc"
sub_model_path="${model_path}/${sub_model_name}"
if [ -d $sub_model_path ]; then
for part in ensemble_train ensemble_validate test; do
infer_data_path="/Youtube-8M/model_predictions/${part}/${model_name}/${sub_model_name}"
if [ ! -d $infer_data_path ]; then
predictions_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_v2_matrix_model/*.tfrecord"
for prev_model in distillchain_v2_video_dcc; do
predictions_data_pattern="${predictions_data_pattern},/Youtube-8M/model_predictions/${part}/${model_name}/${prev_model}/*.tfrecord"
done
echo predictions = "$predictions_data_pattern"
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble-with-predictions.py \
--output_dir="$infer_data_path" \
--train_dir="$sub_model_path" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distill_data_pattern="$predictions_data_pattern" \
--distillation_features=True \
--distillation_as_input=True \
--model=DistillchainCnnDeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_layers=3 \
--deep_chain_relu_cells=256 \
--batch_size=128 \
--file_size=4096
fi
done
fi
elif [ $task == "test-video" ]; then
sub_model_name="distillchain_v2_video_dcc"
sub_model_path="${model_path}/${sub_model_name}"
if [ -d $sub_model_path ]; then
for part in ensemble_train ensemble_validate test; do
infer_data_path="/Youtube-8M/model_predictions/${part}/${model_name}/${sub_model_name}"
if [ ! -d $infer_data_path ]; then
predictions_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_v2_matrix_model/*.tfrecord"
echo predictions = "$predictions_data_pattern"
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble-with-predictions.py \
--output_dir="$infer_data_path" \
--train_dir="$sub_model_path" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--distill_data_pattern="$predictions_data_pattern" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--distillation_features=False \
--distillation_as_input=True \
--model=DistillchainDeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_layers=3 \
--deep_chain_relu_cells=256 \
--batch_size=1024 \
--file_size=4096
fi
done
fi
fi
================================================
FILE: youtube-8m-wangheda/cascade_scripts/distillchain-v2-hybridchain2.sh
================================================
#!/bin/bash
task=$1
model_name="hybridchain2"
model_path="../model/${model_name}"
if [ $task == "train" ]; then
predictions_data_pattern="/Youtube-8M/model_predictions_x32/train/distillation/ensemble_v2_matrix_model/*.tfrecord"
# train model 1: LSTM
sub_model_name="distillchain_v2_lstmparalleloutput"
sub_model_path="${model_path}/${sub_model_name}"
if [ ! -d $sub_model_path ]; then
mkdir -p $sub_model_path
echo predictions = "$predictions_data_pattern"
CUDA_VISIBLE_DEVICES=0 python train-with-predictions.py \
--predictions_data_pattern="$predictions_data_pattern" \
--distillation_features=False \
--distillation_as_input=True \
--train_dir="$sub_model_path" \
--train_data_pattern="/Youtube-8M/data/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=DistillchainLstmParallelFinaloutputModel \
--rnn_swap_memory=True \
--lstm_layers=1 \
--moe_num_mixtures=8 \
--lstm_cells="1024,128" \
--batch_size=128 \
--num_epochs=1 \
--keep_checkpoint_every_n_hours=20 \
--base_learning_rate=0.0008
fi
# inference model 2
infer_data_path="/Youtube-8M/model_predictions_local/train/${model_name}/${sub_model_name}"
if [ ! -d $infer_data_path ]; then
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble-with-predictions.py \
--output_dir="$infer_data_path" \
--train_dir="$sub_model_path" \
--input_data_pattern="/Youtube-8M/data/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distill_data_pattern="$predictions_data_pattern" \
--distillation_features=False \
--distillation_as_input=True \
--model=DistillchainLstmParallelFinaloutputModel \
--rnn_swap_memory=True \
--lstm_layers=1 \
--moe_num_mixtures=8 \
--lstm_cells="1024,128" \
--batch_size=128 \
--file_size=4096
fi
# next predictions pattern
predictions_data_pattern="${predictions_data_pattern},${infer_data_path}/*.tfrecord"
# train model 2: CNN-DCC
sub_model_name="distillchain_v2_cnn_dcc"
sub_model_path="${model_path}/${sub_model_name}"
if [ ! -d $sub_model_path ]; then
mkdir -p $sub_model_path
echo predictions = "$predictions_data_pattern"
CUDA_VISIBLE_DEVICES=0 python train-with-predictions.py \
--predictions_data_pattern="$predictions_data_pattern" \
--distillation_features=False \
--distillation_as_input=True \
--train_dir="$sub_model_path" \
--train_data_pattern="/Youtube-8M/data/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=DistillchainCnnDeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_layers=3 \
--deep_chain_relu_cells=256 \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_type="label,label,label" \
--support_loss_percent=0.05 \
--num_readers=4 \
--batch_size=128 \
--num_epochs=1 \
--keep_checkpoint_every_n_hours=20 \
--base_learning_rate=0.001
fi
elif [ $task == "test-lstm" ]; then
sub_model_name="distillchain_v2_lstmparalleloutput"
sub_model_path="${model_path}/${sub_model_name}"
if [ -d $sub_model_path ]; then
for part in ensemble_train ensemble_validate test; do
infer_data_path="/Youtube-8M/model_predictions/${part}/${model_name}/${sub_model_name}"
if [ ! -d $infer_data_path ]; then
predictions_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_v2_matrix_model/*.tfrecord"
echo predictions = "$predictions_data_pattern"
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble-with-predictions.py \
--output_dir="$infer_data_path" \
--train_dir="$sub_model_path" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distill_data_pattern="$predictions_data_pattern" \
--distillation_features=False \
--distillation_as_input=True \
--model=DistillchainLstmParallelFinaloutputModel \
--rnn_swap_memory=True \
--lstm_layers=1 \
--moe_num_mixtures=8 \
--lstm_cells="1024,128" \
--batch_size=32 \
--file_size=4096
fi
done
fi
elif [ $task == "test-cnn" ]; then
sub_model_name="distillchain_v2_cnn_dcc"
sub_model_path="${model_path}/${sub_model_name}"
if [ -d $sub_model_path ]; then
for part in ensemble_train ensemble_validate test; do
infer_data_path="/Youtube-8M/model_predictions/${part}/${model_name}/${sub_model_name}"
if [ ! -d $infer_data_path ]; then
predictions_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_v2_matrix_model/*.tfrecord"
for prev_model in distillchain_v2_lstmparalleloutput; do
predictions_data_pattern="${predictions_data_pattern},/Youtube-8M/model_predictions_local/${part}/${model_name}/${prev_model}/*.tfrecord"
done
echo predictions = "$predictions_data_pattern"
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble-with-predictions.py \
--output_dir="$infer_data_path" \
--train_dir="$sub_model_path" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distill_data_pattern="$predictions_data_pattern" \
--distillation_features=False \
--distillation_as_input=True \
--model=DistillchainCnnDeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_layers=3 \
--deep_chain_relu_cells=256 \
--batch_size=128 \
--file_size=4096
fi
done
fi
fi
================================================
FILE: youtube-8m-wangheda/cascade_scripts/distillchain-v2-videochain.sh
================================================
#!/bin/bash
task=$1
model_name="videochain"
model_path="../model/${model_name}"
if [ $task == "train" ]; then
predictions_data_pattern='/Youtube-8M/model_predictions_local/train/distillation/ensemble_v2_matrix_model/*.tfrecord'
for i in {1..5}; do
sub_model_name="distillchain_v2_video_dcc_$i"
sub_model_path="${model_path}/${sub_model_name}"
if [ ! -d $sub_model_path ]; then
mkdir -p $sub_model_path
echo predictions = "$predictions_data_pattern"
CUDA_VISIBLE_DEVICES=0 python train-with-predictions.py \
--train_dir="$sub_model_path" \
--train_data_pattern="/Youtube-8M/data/video/train/*.tfrecord" \
--predictions_data_pattern="$predictions_data_pattern" \
--distillation_features=False \
--distillation_as_input=True \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=DistillchainDeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_layers=3 \
--deep_chain_relu_cells=256 \
--data_augmenter=NoiseAugmenter \
--input_noise_level=0.1 \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_type="label,label,label" \
--support_loss_percent=0.05 \
--base_learning_rate=0.01 \
--keep_checkpoint_every_n_hour=5.0 \
--keep_checkpoint_interval=6 \
--num_epochs=2 \
--batch_size=1024
fi
if [ "$i" -ne "5" ]; then
# inference model 1
infer_data_path="/Youtube-8M/model_predictions_local/train/${model_name}/${sub_model_name}"
if [ ! -d $infer_data_path ]; then
echo predictions = "$predictions_data_pattern"
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble-with-predictions.py \
--output_dir="$infer_data_path" \
--train_dir="$sub_model_path" \
--input_data_pattern="/Youtube-8M/data/video/train/*.tfrecord" \
--distill_data_pattern="$predictions_data_pattern" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--distillation_features=False \
--distillation_as_input=True \
--model=DistillchainDeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_layers=3 \
--deep_chain_relu_cells=256 \
--batch_size=1024 \
--file_size=4096
fi
fi
# next predictions pattern
predictions_data_pattern="${predictions_data_pattern},${infer_data_path}/*.tfrecord"
if [ -d $sub_model_path ]; then
for part in ensemble_train ensemble_validate test; do
infer_data_path="/Youtube-8M/model_predictions_local/${part}/${model_name}/${sub_model_name}"
if [ ! -d $infer_data_path ]; then
preensemble_data_pattern="/Youtube-8M/model_predictions_local/${part}/distillation/ensemble_v2_matrix_model/*.tfrecord"
for j in {1..5}; do
if [ "$j" -lt "$i" ]; then
prev_model="distillchain_v2_video_dcc_${j}"
preensemble_data_pattern="${preensemble_data_pattern},/Youtube-8M/model_predictions_local/${part}/${model_name}/${prev_model}/*.tfrecord"
fi
done
echo preensemble_data_pattern = "${preensemble_data_pattern}"
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble-with-predictions.py \
--output_dir="$infer_data_path" \
--train_dir="$sub_model_path" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--distill_data_pattern="$preensemble_data_pattern" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--distillation_features=False \
--distillation_as_input=True \
--model=DistillchainDeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_layers=3 \
--deep_chain_relu_cells=256 \
--batch_size=1024 \
--file_size=4096
fi
done
fi
done
fi
================================================
FILE: youtube-8m-wangheda/cloudml-gpu-distributed.yaml
================================================
trainingInput:
runtimeVersion: "1.0"
scaleTier: CUSTOM
masterType: standard_gpu
workerCount: 2
workerType: standard_gpu
parameterServerCount: 2
parameterServerType: standard
================================================
FILE: youtube-8m-wangheda/cloudml-gpu.yaml
================================================
trainingInput:
scaleTier: CUSTOM
# standard_gpu provides 1 GPU. Change to complex_model_m_gpu for 4 GPUs
masterType: standard_gpu
runtimeVersion: "1.0"
================================================
FILE: youtube-8m-wangheda/data_augmentation.py
================================================
from tensorflow import flags
flags.DEFINE_string("data_augmenter", "DefaultAugmenter",
"how to preprocess feature, defaults to identical, which means no transform")
flags.DEFINE_float("input_noise_level", 0.2,
"the standard deviation of normal noise added to input")
from all_data_augmentation import *
================================================
FILE: youtube-8m-wangheda/data_augmentation_scripts/eval-chaining-video.sh
================================================
GPU_ID=1
EVERY=100
MODEL=DeepCombineChainModel
MODEL_DIR="../model/dataaugmentation_chaining_video"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--moe_num_mixtures=2 \
--num_readers=4 \
--deep_chain_layers=8 \
--deep_chain_relu_cells=40 \
--batch_size=1024 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/data_augmentation_scripts/run-chaining-cnn.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/dataaugmentation_chaining_cnn/" \
--train_data_pattern="/Youtube-8M/data/frame/largetrain/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=CnnDeepCombineChainModel \
--deep_chain_layers=4 \
--deep_chain_relu_cells=128 \
--moe_num_mixtures=4 \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_type="label,label,label,label" \
--support_loss_percent=0.05 \
--num_readers=4 \
--batch_size=40 \
--data_augmenter=HalfAugmenter \
--keep_checkpoint_every_n_hours=1.0 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-wangheda/data_augmentation_scripts/run-chaining-lstm.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/dataaugmentation_chaining_lstm/" \
--train_data_pattern="/Youtube-8M/data/frame/largetrain/*.tfrecord" \
--model=LstmMemoryDeepChainModel \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=200 \
--deep_chain_layers=1 \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--lstm_layers=2 \
--lstm_cells=1024 \
--multitask=True \
--support_type="label" \
--num_supports=4716 \
--label_loss=MultiTaskCrossEntropyLoss \
--base_learning_rate=0.0008 \
--support_loss_percent=0.2 \
--keep_checkpoint_every_n_hours=3.0 \
--rnn_swap_memory=True \
--num_epochs=4 \
--data_augmenter=HalfAugmenter \
--batch_size=40
================================================
FILE: youtube-8m-wangheda/data_augmentation_scripts/run-chaining-video.sh
================================================
CUDA_VISIBLE_DEVICES="0" python train.py \
--train_dir="../model/dataaugmentation_chaining_video/" \
--train_data_pattern="/Youtube-8M/data/frame/largetrain/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=DeepCombineChainModel \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label,label,label,label,label,label" \
--support_loss_percent=0.025 \
--moe_num_mixtures=2 \
--keep_checkpoint_every_n_hours=0.25 \
--num_readers=4 \
--deep_chain_layers=8 \
--deep_chain_relu_cells=40 \
--batch_size=200 \
--data_augmenter=HalfVideoAugmenter \
--num_epochs=5 \
--base_learning_rate=0.005
================================================
FILE: youtube-8m-wangheda/data_augmentation_scripts/run-multiple-attention-pooling-positional-embedding.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/dataaugmentation_attention_pooling" \
--train_data_pattern="/Youtube-8M/data/frame/largetrain/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=LstmPositionalAttentionMaxPoolingModel \
--moe_num_mixtures=8 \
--lstm_attentions=1 \
--positional_embedding_size=32 \
--rnn_swap_memory=True \
--base_learning_rate=0.001 \
--num_readers=4 \
--num_epochs=6 \
--batch_size=40 \
--data_augmenter=HalfAugmenter \
--keep_checkpoint_every_n_hour=2.0
================================================
FILE: youtube-8m-wangheda/data_augmentation_scripts/run-multiscale-cnn-lstm-model.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/dataaugmentation_multiscale_cnn_lstm" \
--train_data_pattern='/Youtube-8M/data/frame/largetrain/*.tfrecord' \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model='MultiscaleCnnLstmModel' \
--multiscale_cnn_lstm_layers=4 \
--moe_num_mixtures=4 \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_loss_percent=1.0 \
--support_type="label,label,label,label" \
--is_training=True \
--num_readers=4 \
--num_epochs=5 \
--batch_size=40 \
--data_augmenter=HalfAugmenter \
--base_learning_rate=0.001 \
--keep_checkpoint_every_n_hour=2.0
================================================
FILE: youtube-8m-wangheda/data_augmentation_scripts/run-parallel-lstm-memory.sh
================================================
CUDA_VISIBLE_DEVICES=1 python train.py \
--train_dir="../model/dataaugmentation_parallel_lstm_memory" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--train_data_pattern="/Youtube-8M/data/frame/largetrain/*.tfrecord" \
--lstm_cells="1024,128" \
--moe_num_mixtures=8 \
--model=LstmParallelMemoryModel \
--rnn_swap_memory=True \
--num_readers=4 \
--batch_size=40 \
--data_augmenter=HalfAugmenter \
--num_epochs=5 \
--base_learning_rate=0.0008 \
--keep_checkpoint_every_n_hour=2.0
================================================
FILE: youtube-8m-wangheda/eval.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for evaluating Tensorflow models on the YouTube-8M dataset."""
import time
import eval_util
import losses
import frame_level_models
import video_level_models
import feature_transform
import readers
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import utils
FLAGS = flags.FLAGS
if __name__ == "__main__":
# Dataset flags.
flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
"The directory to load the model files from. "
"The tensorboard metrics files are also saved to this "
"directory.")
flags.DEFINE_string("model_checkpoint_path", "",
"The file to load the model files from. ")
flags.DEFINE_string(
"eval_data_pattern", "",
"File glob defining the evaluation dataset in tensorflow.SequenceExample "
"format. The SequenceExamples are expected to have an 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
flags.DEFINE_string(
"distill_data_pattern", None,
"File glob defining the distillation data pattern")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024", "Length of the feature vectors.")
# Model flags.
flags.DEFINE_bool(
"frame_features", False,
"If set, then --eval_data_pattern must be frame-level features. "
"Otherwise, --eval_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_string(
"model", "LogisticModel",
"Which architecture to use for the model. Options include 'Logistic', "
"'SingleMixtureMoe', and 'TwoLayerSigmoid'. See aggregated_models.py and "
"frame_level_models.py for the model definitions.")
flags.DEFINE_integer("batch_size", 1024,
"How many examples to process per batch.")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Loss computed on validation data")
# Other flags.
flags.DEFINE_integer("num_readers", 8,
"How many threads to use for reading input files.")
flags.DEFINE_boolean("run_once", False, "Whether to run eval only once.")
flags.DEFINE_integer("top_k", 20, "How many predictions to output per video.")
flags.DEFINE_bool(
"multitask", False,
"Whether to consider support_predictions")
flags.DEFINE_bool(
"dropout", False,
"Whether to consider dropout")
flags.DEFINE_float("keep_prob", 1.0,
"probability to keep output (used in dropout, keep it unchanged in validationg and test)")
flags.DEFINE_float("noise_level", 0.0,
"standard deviation of noise (added to hidden nodes)")
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def get_input_evaluation_tensors(reader,
data_pattern,
batch_size=1024):
logging.info("Using batch size of " + str(batch_size) + " for evaluation.")
with tf.name_scope("eval_input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find the evaluation files.")
logging.info("number of evaluation files: " + str(len(files)))
files.sort()
filename_queue = tf.train.string_input_producer(
files, shuffle=False, num_epochs=1)
eval_data = reader.prepare_reader(filename_queue)
return tf.train.batch(
eval_data,
batch_size=batch_size,
capacity=3 * batch_size,
allow_smaller_final_batch=True,
enqueue_many=True)
def build_graph(reader,
model,
eval_data_pattern,
label_loss_fn,
batch_size=1024,
transformer_class=feature_transform.DefaultTransformer,
distill_reader=None,
num_readers=1):
"""Creates the Tensorflow graph for evaluation.
Args:
reader: The data file reader. It should inherit from BaseReader.
model: The core model (e.g. logistic or neural net). It should inherit
from BaseModel.
eval_data_pattern: glob path to the evaluation data files.
label_loss_fn: What kind of loss to apply to the model. It should inherit
from BaseLoss.
batch_size: How many examples to process at a time.
num_readers: How many threads to use for I/O operations.
"""
global_step = tf.Variable(0, trainable=False, name="global_step")
video_id_batch, model_input_raw, labels_batch, num_frames = get_input_evaluation_tensors( # pylint: disable=g-line-too-long
reader,
eval_data_pattern,
batch_size=batch_size)
tf.summary.histogram("model_input_raw", model_input_raw)
if distill_reader is not None:
unused_video_id_batch, distill_input_raw, unused_labels_batch, unused_num_frames = get_input_evaluation_tensors( # pylint: disable=g-line-too-long
distill_reader,
FLAGS.distill_data_pattern,
batch_size=batch_size)
feature_dim = len(model_input_raw.get_shape()) - 1
# Normalize input features.
feature_transformer = transformer_class()
model_input, num_frames = feature_transformer.transform(model_input_raw, num_frames=num_frames)
with tf.name_scope("model"):
if FLAGS.noise_level > 0:
noise_level_tensor = tf.placeholder_with_default(0.0, shape=[], name="noise_level")
else:
noise_level_tensor = None
if distill_reader is not None:
distillation_predictions = distill_input_raw
else:
distillation_predictions = None
if FLAGS.dropout:
keep_prob_tensor = tf.placeholder_with_default(1.0, shape=[], name="keep_prob")
result = model.create_model(model_input,
num_frames=num_frames,
vocab_size=reader.num_classes,
labels=labels_batch,
dropout=FLAGS.dropout,
keep_prob=keep_prob_tensor,
distillation_predictions=distillation_predictions,
is_training=False)
else:
result = model.create_model(model_input,
num_frames=num_frames,
vocab_size=reader.num_classes,
labels=labels_batch,
distillation_predictions=distillation_predictions,
is_training=False)
predictions = result["predictions"]
tf.summary.histogram("model_activations", predictions)
if "loss" in result.keys():
label_loss = result["loss"]
else:
if FLAGS.multitask:
support_predictions = result["support_predictions"]
label_loss = label_loss_fn.calculate_loss(predictions, support_predictions, labels_batch)
else:
label_loss = label_loss_fn.calculate_loss(predictions, labels_batch)
tf.add_to_collection("global_step", global_step)
tf.add_to_collection("loss", label_loss)
tf.add_to_collection("predictions", predictions)
tf.add_to_collection("input_batch", model_input)
tf.add_to_collection("video_id_batch", video_id_batch)
tf.add_to_collection("num_frames", num_frames)
tf.add_to_collection("labels", tf.cast(labels_batch, tf.float32))
tf.add_to_collection("summary_op", tf.summary.merge_all())
if FLAGS.dropout:
tf.add_to_collection("keep_prob", keep_prob_tensor)
if FLAGS.noise_level > 0:
tf.add_to_collection("noise_level", noise_level_tensor)
def evaluation_loop(video_id_batch, prediction_batch, label_batch, loss,
summary_op, saver, summary_writer, evl_metrics,
last_global_step_val):
"""Run the evaluation loop once.
Args:
video_id_batch: a tensor of video ids mini-batch.
prediction_batch: a tensor of predictions mini-batch.
label_batch: a tensor of label_batch mini-batch.
loss: a tensor of loss for the examples in the mini-batch.
summary_op: a tensor which runs the tensorboard summary operations.
saver: a tensorflow saver to restore the model.
summary_writer: a tensorflow summary_writer
evl_metrics: an EvaluationMetrics object.
last_global_step_val: the global step used in the previous evaluation.
Returns:
The global_step used in the latest model.
"""
global_step_val = -1
with tf.Session() as sess:
if FLAGS.model_checkpoint_path:
checkpoint = FLAGS.model_checkpoint_path
else:
checkpoint = tf.train.latest_checkpoint(FLAGS.train_dir)
if checkpoint:
logging.info("Loading checkpoint for eval: " + checkpoint)
# Restores from checkpoint
saver.restore(sess, checkpoint)
# Assuming model_checkpoint_path looks something like:
# /my-favorite-path/yt8m_train/model.ckpt-0, extract global_step from it.
global_step_val = checkpoint.split("/")[-1].split("-")[-1]
else:
logging.info("No checkpoint file found.")
return global_step_val
if global_step_val == last_global_step_val:
logging.info("skip this checkpoint global_step_val=%s "
"(same as the previous one).", global_step_val)
return global_step_val
sess.run([tf.local_variables_initializer()])
# Start the queue runners.
if FLAGS.dropout:
keep_prob_tensor = tf.get_collection("keep_prob")[0]
if FLAGS.noise_level > 0:
noise_level_tensor = tf.get_collection("noise_level")[0]
fetches = [video_id_batch, prediction_batch, label_batch, loss, summary_op]
coord = tf.train.Coordinator()
try:
threads = []
for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
threads.extend(qr.create_threads(
sess, coord=coord, daemon=True,
start=True))
logging.info("enter eval_once loop global_step_val = %s. ",
global_step_val)
evl_metrics.clear()
examples_processed = 0
while not coord.should_stop():
batch_start_time = time.time()
custom_feed = {}
if FLAGS.dropout:
custom_feed[keep_prob_tensor] = FLAGS.keep_prob
if FLAGS.noise_level > 0:
custom_feed[noise_level_tensor] = FLAGS.noise_level
_, predictions_val, labels_val, loss_val, summary_val = sess.run(
fetches, feed_dict=custom_feed)
seconds_per_batch = time.time() - batch_start_time
example_per_second = labels_val.shape[0] / seconds_per_batch
examples_processed += labels_val.shape[0]
iteration_info_dict = evl_metrics.accumulate(predictions_val,
labels_val, loss_val)
iteration_info_dict["examples_per_second"] = example_per_second
iterinfo = utils.AddGlobalStepSummary(
summary_writer,
global_step_val,
iteration_info_dict,
summary_scope="Eval")
logging.info("examples_processed: %d | %s", examples_processed,
iterinfo)
except tf.errors.OutOfRangeError as e:
logging.info(
"Done with batched inference. Now calculating global performance "
"metrics.")
# calculate the metrics for the entire epoch
epoch_info_dict = evl_metrics.get()
epoch_info_dict["epoch_id"] = global_step_val
summary_writer.add_summary(summary_val, global_step_val)
epochinfo = utils.AddEpochSummary(
summary_writer,
global_step_val,
epoch_info_dict,
summary_scope="Eval")
logging.info(epochinfo)
evl_metrics.clear()
except Exception as e: # pylint: disable=broad-except
logging.info("Unexpected exception: " + str(e))
coord.request_stop(e)
coord.request_stop()
coord.join(threads, stop_grace_period_secs=10)
return global_step_val
def evaluate():
tf.set_random_seed(0) # for reproducibility
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if FLAGS.distill_data_pattern is not None:
distill_reader = readers.YT8MAggregatedFeatureReader(feature_names=["predictions"],
feature_sizes=[4716])
else:
distill_reader = None
model = find_class_by_name(FLAGS.model,
[frame_level_models, video_level_models])()
label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])()
transformer_class = find_class_by_name(FLAGS.feature_transformer, [feature_transform])
if FLAGS.eval_data_pattern is "":
raise IOError("'eval_data_pattern' was not specified. " +
"Nothing to evaluate.")
build_graph(
reader=reader,
model=model,
eval_data_pattern=FLAGS.eval_data_pattern,
label_loss_fn=label_loss_fn,
num_readers=FLAGS.num_readers,
transformer_class=transformer_class,
distill_reader=distill_reader,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
video_id_batch = tf.get_collection("video_id_batch")[0]
prediction_batch = tf.get_collection("predictions")[0]
label_batch = tf.get_collection("labels")[0]
loss = tf.get_collection("loss")[0]
summary_op = tf.get_collection("summary_op")[0]
saver = tf.train.Saver(tf.global_variables())
summary_writer = tf.summary.FileWriter(
FLAGS.train_dir, graph=tf.get_default_graph())
evl_metrics = eval_util.EvaluationMetrics(reader.num_classes, FLAGS.top_k)
last_global_step_val = -1
while True:
last_global_step_val = evaluation_loop(video_id_batch, prediction_batch,
label_batch, loss, summary_op,
saver, summary_writer, evl_metrics,
last_global_step_val)
if FLAGS.run_once:
break
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
evaluate()
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-wangheda/eval.sh
================================================
GPU_ID=0
EVERY=500
MODEL=LstmModel
MODEL_DIR="../model/attentionlstm_moe4"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M-validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--model=$MODEL \
--lstm_cells=1024 \
--moe_num_mixtures=4 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-att-lstm.sh
================================================
GPU_ID=1
EVERY=500
MODEL=LstmAttentionLstmModel
MODEL_DIR="../model/lstmattlstm1024_moe8"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M-validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--model=$MODEL \
--lstm_cells=1024 \
--lstm_layers=1 \
--moe_num_mixtures=8 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-att.sh
================================================
GPU_ID=1
EVERY=500
MODEL=LstmAttentionModel
MODEL_DIR="../model/attentionlstm_moe4"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M-validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--model=$MODEL \
--lstm_cells=1024 \
--moe_num_mixtures=4 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-bi-uni-lstm.sh
================================================
GPU_ID=1
EVERY=500
MODEL=BiUniLstmModel
MODEL_DIR="../model/biunilstm1024_moe4"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=32 \
--model=$MODEL \
--lstm_cells=1024 \
--moe_num_mixtures=4 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-chain-model-relu.sh
================================================
start=$1
GPU_ID=1
MODEL=ChainMainReluMoeModel
MODEL_DIR="../model/video_chain_main_relu_moe16_ce"
for checkpoint in $(for filename in $MODEL_DIR/model.ckpt-*.meta; do echo $filename | grep -o "ckpt-[0123456789]*.meta" | cut -d '-' -f 2 | cut -d '.' -f 1; done | sort -n); do
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=256 \
--model=$MODEL \
--moe_num_mixtures=16 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-chain-model-suprelu.sh
================================================
start=$1
GPU_ID=1
MODEL=ChainSupportReluMoeModel
MODEL_DIR="../model/video_chain_support_relu_moe16_ce"
for checkpoint in $(for filename in $MODEL_DIR/model.ckpt-*.meta; do echo $filename | grep -o "ckpt-[0123456789]*.meta" | cut -d '-' -f 2 | cut -d '.' -f 1; done | sort -n); do
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=256 \
--model=$MODEL \
--moe_num_mixtures=16 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-chain-moe-0.4.sh
================================================
GPU_ID="1"
EVERY=500
MODEL=ChainMoeModel
MODEL_DIR="../model/video_chain_moe16_ce_0.4"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=256 \
--model=$MODEL \
--moe_num_mixtures=16 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-chain-moe-freq.sh
================================================
GPU_ID="1"
EVERY=100
MODEL=ChainSupportReluMoeModel
MODEL_DIR="../model/video_chain_moe16_ce_freq"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=256 \
--num_supports=200 \
--support_type="frequent" \
--model=$MODEL \
--moe_num_mixtures=16 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-chain-moe-suprelu-vert+freq.sh
================================================
start=$1
GPU_ID=1
MODEL=ChainSupportReluMoeModel
MODEL_DIR="../model/video_chain_support_relu_moe16_ce_vert+freq"
for checkpoint in $(for filename in $MODEL_DIR/model.ckpt-*.meta; do echo $filename | grep -o "ckpt-[0123456789]*.meta" | cut -d '-' -f 2 | cut -d '.' -f 1; done | sort -n); do
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=256 \
--support_type="vertical,frequent" \
--num_supports=225 \
--num_verticals=25 \
--num_frequents=200 \
--model=$MODEL \
--moe_num_mixtures=16 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-chain-moe.sh
================================================
GPU_ID="1"
EVERY=500
MODEL=ChainMoeModel
MODEL_DIR="../model/video_chain_moe16_ce"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=256 \
--model=$MODEL \
--moe_num_mixtures=16 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-cnn-deep-combine-chain.sh
================================================
GPU_ID="1"
EVERY=1000
MODEL=CnnDeepCombineChainModel
MODEL_DIR="../model/cnn_deep_combine_chain"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=64 \
--model=$MODEL \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label,label" \
--support_loss_percent=0.05 \
--moe_num_mixtures=4 \
--deep_chain_layers=4 \
--deep_chain_relu_cells=128 \
--rnn_swap_memory=True \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-cnn-model.sh
================================================
GPU_ID="0"
EVERY=4000
MODEL=CnnModel
MODEL_DIR="../model/cnn_model"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--cnn_num_filters=512 \
--moe_num_mixtures=4 \
--num_readers=4 \
--batch_size=128 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-dbof.sh
================================================
GPU_ID=1
EVERY=1000
MODEL=DbofModel
MODEL_DIR="../model/dbof_model"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--batch_size=128 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-deep-cnn-deep-combine-chain.sh
================================================
GPU_ID=0
EVERY=1000
MODEL=DeepCnnDeepCombineChainModel
MODEL_DIR="../model/deep_cnn_deep_combine_chain"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--moe_num_mixtures=4 \
--deep_chain_layers=2 \
--deep_chain_relu_cells=256 \
--deep_cnn_base_size=128 \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label" \
--support_loss_percent=0.1 \
--batch_size=256 \
--model=$MODEL \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-distill-video-dcc-noise-scene1.sh
================================================
percent="$1"
start=0
GPU_ID=0
EVERY=200
MODEL=DeepCombineChainModel
MODEL_DIR="../model/distillation_video_dcc_noise/scene1_percent_${percent}"
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--deep_chain_relu_cells=256 \
--deep_chain_layers=4 \
--moe_num_mixtures=4 \
--batch_size=1024 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-distill-video-dcc-noise-scene2.sh
================================================
percent="$1"
start=0
GPU_ID=0
EVERY=200
MODEL=DeepCombineChainModel
MODEL_DIR="../model/distillation_video_dcc_noise/scene2_percent_${percent}"
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--deep_chain_relu_cells=256 \
--deep_chain_layers=4 \
--moe_num_mixtures=4 \
--batch_size=1024 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-distillchain-cnn-dcc.sh
================================================
start=$1
GPU_ID=0
EVERY=1000
MODEL=DistillchainCnnDeepCombineChainModel
MODEL_DIR="../model/distillchain_cnn_dcc"
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--distill_data_pattern="/Youtube-8M/model_predictions/validatea/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_as_input=True \
--model=$MODEL \
--deep_chain_layers=3 \
--deep_chain_relu_cells=256 \
--moe_num_mixtures=4 \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_type="label,label,label" \
--support_loss_percent=0.05 \
--batch_size=128 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-distillchain-lstmcnn.sh
================================================
start=$1
GPU_ID=1
EVERY=2000
MODEL=DistillchainLstmCnnDeepCombineChainModel
MODEL_DIR="../model/distillchain_lstmcnn_dcc"
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--distill_data_pattern="/Youtube-8M/model_predictions/validatea/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_as_input=True \
--model=$MODEL \
--deep_chain_layers=2 \
--deep_chain_relu_cells=128 \
--distillchain_relu_cells=256 \
--moe_num_mixtures=4 \
--lstm_layers=1 \
--lstm_cells="1024,128" \
--rnn_swap_memory=True \
--batch_size=32 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-distillchain-lstmparalleloutput.sh
================================================
start=$1
GPU_ID=0
EVERY=2000
MODEL=DistillchainLstmParallelFinaloutputModel
MODEL_DIR="../model/distillchain_lstmparalleloutput"
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--distill_data_pattern="/Youtube-8M/model_predictions/validatea/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_as_input=True \
--model=$MODEL \
--rnn_swap_memory=True \
--lstm_cells="1024,128" \
--moe_num_mixtures=8 \
--batch_size=128 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-distillchain-multilstm.sh
================================================
start=$1
GPU_ID=1
EVERY=100
MODEL=DistillchainLstmMemoryDeepCombineChainModel
MODEL_DIR="../model/distillchain_multilstm_dcc"
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--distill_data_pattern="/Youtube-8M/model_predictions/validatea/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_as_input=True \
--model=$MODEL \
--lstm_layers=1 \
--lstm_cells=1024 \
--moe_num_mixtures=4 \
--distillation_relu_cells=256 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=2 \
--rnn_swap_memory=True \
--batch_size=32 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-distillchain-v2-boostinglstmparalleloutput.sh
================================================
start=$1
GPU_ID=1
EVERY=2000
MODEL=DistillchainLstmParallelFinaloutputModel
MODEL_DIR="../model/distillchain_v2_boost_lstmparalleloutput"
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--distill_data_pattern="/Youtube-8M/model_predictions/validatea/distillation/ensemble_v2_matrix_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=False \
--distillation_as_input=True \
--model=$MODEL \
--lstm_layers=1 \
--rnn_swap_memory=True \
--lstm_cells="1024,128" \
--moe_num_mixtures=8 \
--batch_size=32 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-distillchain-v2-lstmattention8max.sh
================================================
start=$1
GPU_ID=0
EVERY=2000
MODEL=DistillchainLstmAttentionMaxPoolingModel \
MODEL_DIR="../model/distillchain_v2_lstmattention8max"
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--distill_data_pattern="/Youtube-8M/model_predictions/validatea/distillation/ensemble_v2_matrix_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=False \
--distillation_as_input=True \
--model=$MODEL \
--moe_num_mixtures=8 \
--lstm_attentions=8 \
--lstm_cells=1024 \
--rnn_swap_memory=True \
--moe_num_mixtures=8 \
--batch_size=128 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-distillchain-v2-lstmcnn.sh
================================================
start=$1
GPU_ID=0
EVERY=2000
MODEL=DistillchainLstmCnnDeepCombineChainModel
MODEL_DIR="../model/distillchain_v2_lstmcnn_dcc"
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--distill_data_pattern="/Youtube-8M/model_predictions/validatea/distillation/ensemble_v2_matrix_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=False \
--distillation_as_input=True \
--model=$MODEL \
--deep_chain_layers=2 \
--deep_chain_relu_cells=128 \
--distillchain_relu_cells=256 \
--moe_num_mixtures=4 \
--lstm_layers=1 \
--lstm_cells="1024,128" \
--rnn_swap_memory=True \
--batch_size=128 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-distillchain-v2-lstmparalleloutput.sh
================================================
start=$1
GPU_ID=1
EVERY=2000
MODEL=DistillchainLstmParallelFinaloutputModel
MODEL_DIR="../model/distillchain_v2_lstmparalleloutput"
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--distill_data_pattern="/Youtube-8M/model_predictions/validatea/distillation/ensemble_v2_matrix_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=False \
--distillation_as_input=True \
--model=$MODEL \
--rnn_swap_memory=True \
--lstm_cells="1024,128" \
--moe_num_mixtures=4 \
--batch_size=32 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-distillchain-v2-multilstm.sh
================================================
start=$1
GPU_ID=0
EVERY=500
MODEL=DistillchainLstmMemoryDeepCombineChainModel
MODEL_DIR="../model/distillchain_v2_multilstm_dcc"
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--distill_data_pattern="/Youtube-8M/model_predictions/validatea/distillation/ensemble_v2_matrix_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=False \
--distillation_as_input=True \
--model=$MODEL \
--lstm_layers=1 \
--lstm_cells=1024 \
--moe_num_mixtures=4 \
--distillation_relu_cells=256 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=2 \
--rnn_swap_memory=True \
--batch_size=128 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-distillchain-v2-multiscale-cnnlstm.sh
================================================
start=$1
GPU_ID=0
EVERY=2000
MODEL=DistillchainMultiscaleCnnLstmModel
MODEL_DIR="../model/distillchain_v2_multiscale_cnnlstm"
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--distill_data_pattern="/Youtube-8M/model_predictions/validatea/distillation/ensemble_v2_matrix_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=False \
--distillation_as_input=True \
--model=$MODEL \
--multiscale_cnn_lstm_layers=3 \
--moe_num_mixtures=4 \
--rnn_swap_memory=True \
--is_training=False \
--batch_size=128 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-distillchain-v2-video-dcc.sh
================================================
noise_level=0.15
start=$1
GPU_ID=0
EVERY=500
MODEL=DistillchainDeepCombineChainModel
MODEL_DIR="../model/distillchain_v2_video_dcc"
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--distill_data_pattern="/Youtube-8M/model_predictions/validatea/distillation/ensemble_v2_matrix_model/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--deep_chain_relu_cells=256 \
--deep_chain_layers=4 \
--moe_num_mixtures=4 \
--batch_size=1024 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-distillchain-video-dcc.sh
================================================
noise_level=0.15
start=$1
GPU_ID=0
EVERY=100
MODEL=DistillchainDeepCombineChainModel
MODEL_DIR="../model/distillchain_video_dcc"
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--distill_data_pattern="/Youtube-8M/model_predictions/validatea/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label,label" \
--num_supports=18864 \
--support_loss_percent=0.05 \
--model=$MODEL \
--deep_chain_relu_cells=200 \
--deep_chain_layers=4 \
--moe_num_mixtures=4 \
--batch_size=1024 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-frame-seg.sh
================================================
GPU_ID="1"
EVERY=1000
MODEL=FrameSegModel
MODEL_DIR="../model/frame_seg_model"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=32 \
--feature_transformer=IdenticalTransformer \
--model=$MODEL \
--moe_num_mixtures=16 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-framehop-lstmmem.sh
================================================
GPU_ID=1
EVERY=500
MODEL=FramehopLstmMemoryModel
MODEL_DIR="../model/framehop_lstm"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M-validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=64 \
--model=$MODEL \
--deep_chain_layers=4 \
--deep_chain_relu_cells=256 \
--feature_transformer=IdenticalTransformer \
--lstm_layers=2 \
--lstm_cells="512,64" \
--moe_num_mixtures=4 \
--rnn_swap_memory=True \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-layer-chain-moe8-freq.sh
================================================
GPU_ID="1"
EVERY=500
MODEL=ChainSupportReluMoeModel
MODEL_DIR="../model/layer_chain_moe8_freq"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/layer/validate/validate*0.tfrecord" \
--frame_features=False \
--feature_names="layer" \
--feature_sizes="2048" \
--batch_size=128 \
--num_supports=200 \
--support_type="frequent" \
--model=$MODEL \
--moe_num_mixtures=8 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-layer-moe-vert.sh
================================================
GPU_ID="1"
EVERY=500
MODEL=ChainSupportReluMoeModel
MODEL_DIR="../model/layer_chain_moe8_vert"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/layer/validate/validate*0.tfrecord" \
--frame_features=False \
--feature_names="layer" \
--feature_sizes="2048" \
--batch_size=128 \
--num_supports=25 \
--support_type="vertical" \
--model=$MODEL \
--moe_num_mixtures=8 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstm-attention-8max.sh
================================================
GPU_ID="1"
EVERY=200
MODEL=LstmAttentionMaxPoolingModel
MODEL_DIR="../model/lstmattention8max_bagging/base_model"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--moe_num_mixtures=8 \
--lstm_attentions=8 \
--rnn_swap_memory=True \
--batch_size=32 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstm-cnn-deep-combine-chain.sh
================================================
GPU_ID=0
EVERY=500
MODEL=LstmCnnDeepCombineChainModel
MODEL_DIR="../model/lstm_cnn_deep_combine_chain"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--lstm_layers=1 \
--lstm_cells="1024,128" \
--deep_chain_layers=3 \
--deep_chain_relu_cells=128 \
--moe_num_mixtures=4 \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label" \
--support_loss_percent=0.05 \
--batch_size=32 \
--rnn_swap_memory=True \
--num_readers=4 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstm-look-back.sh
================================================
GPU_ID="1"
EVERY=1000
MODEL=LstmLookBackModel
MODEL_DIR="../model/lstmlookback1024_moe8"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=32 \
--model=$MODEL \
--moe_num_mixtures=8 \
--lstm_layers=2 \
--lstm_cells="1024,128" \
--lstm_look_back=3 \
--rnn_swap_memory=True \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstm-positional-attention-8max.sh
================================================
GPU_ID="1"
EVERY=1000
MODEL=LstmPositionalAttentionMaxPoolingModel
MODEL_DIR="../x27_model/lstm_positional_attention8max"
start=70000
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--moe_num_mixtures=8 \
--lstm_attentions=8 \
--positional_embedding_size=32 \
--rnn_swap_memory=True \
--batch_size=32 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstmmem-augmenter.sh
================================================
GPU_ID=0
EVERY=1000
MODEL=LstmMemoryModel
MODEL_DIR="../model/lstmmemory1024_moe8_augment"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--moe_num_mixtures=8 \
--lstm_cells=1024 \
--lstm_layers=2 \
--batch_size=64 \
--rnn_swap_memory=True \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstmmem-chain-freq.sh
================================================
GPU_ID=1
EVERY=500
MODEL=LstmMemoryModel
MODEL_DIR="../model/lstmmemory1024_moe8_chain_freq"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=64 \
--model=LstmMemoryModel \
--video_level_classifier_model=ChainSupportReluMoeModel \
--lstm_layers=2 \
--lstm_cells=1024 \
--multitask=True \
--support_type="frequent" \
--num_supports=200 \
--num_frequents=200 \
--label_loss=MultiTaskCrossEntropyLoss \
--base_learning_rate=0.0008 \
--vertical_loss_percent=0.4 \
--moe_num_mixtures=8 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstmmem-chain.sh
================================================
GPU_ID=1
EVERY=500
MODEL=LstmMemoryChainModel
MODEL_DIR="../model/lstmmemory1024_moe8_chain"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M-validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--model=$MODEL \
--lstm_layers=2 \
--lstm_cells=1024 \
--multitask=True \
--num_supports=25 \
--num_verticals=25 \
--vertical_file="resources/vertical.tsv" \
--label_loss=MultiTaskCrossEntropyLoss \
--base_learning_rate=0.0008 \
--vertical_loss_percent=0.4 \
--moe_num_mixtures=8 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstmmem-cnnlstm.sh
================================================
GPU_ID="1"
EVERY=1000
MODEL=CnnLstmMemoryModel
MODEL_DIR="../model/cnnlstmmemory1024_moe8"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=32 \
--model=$MODEL \
--moe_num_mixtures=8 \
--lstm_layers=2 \
--rnn_swap_memory=True \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstmmem-deep-chain.sh
================================================
GPU_ID=1
EVERY=500
MODEL=LstmMemoryModel
MODEL_DIR="../model/lstmmemory1024_moe4_deep_chain"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/run/shm/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=LstmMemoryModel \
--lstm_layers=2 \
--lstm_cells=1024 \
--video_level_classifier_model=DeepChainModel \
--deep_chain_relu_cells=200 \
--deep_chain_layers=1 \
--moe_num_mixtures=4 \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label" \
--num_supports=4716 \
--support_loss_percent=0.1 \
--batch_size=128 \
--num_readers=4 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstmmem-deep-combine-chain-length.sh
================================================
GPU_ID="0"
EVERY=1000
MODEL=LstmMemoryModel
MODEL_DIR="../model/lstmmem1024_deep_combine_chain_length" \
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=LstmMemoryModel \
--video_level_classifier_model=DeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=1 \
--deep_chain_use_length=True \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label" \
--num_supports=4716 \
--support_loss_percent=0.05 \
--batch_size=128 \
--model=$MODEL \
--lstm_layers=2 \
--rnn_swap_memory=True \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstmmem-dropout.sh
================================================
GPU_ID=0
EVERY=1000
MODEL=LstmMemoryModel
MODEL_DIR="../model/lstmmemory1024_moe8_dropout"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--moe_num_mixtures=8 \
--lstm_cells=1024 \
--lstm_layers=2 \
--dropout=True \
--batch_size=64 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstmmem-feature.sh
================================================
GPU_ID="1"
EVERY=100
MODEL=LstmMemoryModel
MODEL_DIR="../model/lstmmemory_moe8_feature"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--feature_transformer=EngineerTransformer \
--engineer_types="identical,avg,std,diff" \
--batch_size=32 \
--model=$MODEL \
--moe_num_mixtures=8 \
--lstm_layers=2 \
--rnn_swap_memory=True \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstmmem-input-chain.sh
================================================
GPU_ID=1
EVERY=500
MODEL=LstmMemoryInputChainModel
MODEL_DIR="../model/lstmmemory1024_moe8_input_chain"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=64 \
--model=$MODEL \
--lstm_layers=2 \
--lstm_cells=1024 \
--multitask=True \
--moe_num_mixtures=8 \
--num_supports=25 \
--num_verticals=25 \
--vertical_file="resources/vertical.tsv" \
--label_loss=MultiTaskCrossEntropyLoss \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstmmem-input-noise.sh
================================================
noise_level=$1
start=$2
EVERY=1000
MODEL=LstmMemoryModel
MODEL_DIR="../model/lstmmemory1024_moe8_input_noise/noise_level_$noise_level"
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=LstmMemoryModel \
--lstm_layers=2 \
--lstm_cells=1024 \
--moe_num_mixtures=8 \
--batch_size=128 \
--num_readers=4 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstmmem-l2norm.sh
================================================
GPU_ID=1
EVERY=500
MODEL=LstmMemoryNormalizationModel
MODEL_DIR="../model/lstmmemory1024_moe8_l2norm"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M-validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--model=$MODEL \
--lstm_normalization="l2_normalize" \
--feature_transformer="IdenticalTransformer" \
--lstm_cells=1024 \
--lstm_layers=2 \
--moe_num_mixtures=8 \
--num_readers=1 \
--rnn_swap_memory=True \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstmmem-layernorm.sh
================================================
GPU_ID=0
EVERY=500
MODEL=LayerNormLstmMemoryModel
MODEL_DIR="../model/layernormlstmmemory1024_moe8"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M-validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--model=$MODEL \
--lstm_cells=1024 \
--lstm_layers=2 \
--moe_num_mixtures=8 \
--num_readers=1 \
--rnn_swap_memory=True \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstmmem-lowres.sh
================================================
GPU_ID="0"
EVERY=1000
MODEL=LstmMemoryModel
MODEL_DIR="../model/lstmmemory_moe8_lowres"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=32 \
--model=$MODEL \
--moe_num_mixtures=8 \
--lstm_layers=2 \
--feature_transformer=ResolutionTransformer \
--rnn_swap_memory=True \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstmmem-no-transform.sh
================================================
GPU_ID=1
EVERY=500
MODEL=LstmMemoryModel
MODEL_DIR="../model/lstmmemory1024_moe8_no_transform"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--feature_transformer=IdenticalTransformer \
--batch_size=64 \
--model=$MODEL \
--lstm_cells=1024 \
--moe_num_mixtures=8 \
--rnn_swap_memory=True \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstmmem-noise.sh
================================================
GPU_ID=1
EVERY=1000
MODEL=LstmMemoryModel
MODEL_DIR="../model/lstmmemory1024_moe8_noise"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--moe_num_mixtures=8 \
--lstm_cells=1024 \
--lstm_layers=2 \
--batch_size=64 \
--rnn_swap_memory=True \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstmmem-parallel.sh
================================================
start=$1
GPU_ID=0
MODEL=LstmParallelMemoryModel
MODEL_DIR="../model/lstmparallelmemory1024_moe8"
for checkpoint in $(for filename in $MODEL_DIR/model.ckpt-*.meta; do echo $filename | grep -o "ckpt-[0123456789]*.meta" | cut -d '-' -f 2 | cut -d '.' -f 1; done | sort -n); do
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--lstm_cells="1024,128" \
--batch_size=128 \
--model=$MODEL \
--moe_num_mixtures=8 \
--rnn_swap_memory=True \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstmmem-shortcut-chain-freq.sh
================================================
GPU_ID=1
EVERY=500
MODEL=LstmMemoryModel
MODEL_DIR="../model/lstmmemory1024_moe8_shortcut_chain_freq"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=64 \
--model=LstmMemoryModel \
--video_level_classifier_model=ShortcutChainSupportReluMoeModel \
--lstm_layers=2 \
--lstm_cells=1024 \
--multitask=True \
--support_type="frequent" \
--num_supports=200 \
--num_frequents=200 \
--label_loss=MultiTaskCrossEntropyLoss \
--base_learning_rate=0.0008 \
--vertical_loss_percent=0.4 \
--moe_num_mixtures=8 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstmmem2048.sh
================================================
GPU_ID=0
EVERY=1000
MODEL=LstmMemoryModel
MODEL_DIR="../model/lstmmemory2048_moe4_batch128"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--moe_num_mixtures=4 \
--lstm_cells=2048 \
--lstm_layers=2 \
--batch_size=64 \
--rnn_swap_memory=True \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstmmemory-audio.sh
================================================
GPU_ID="0"
EVERY=2000
MODEL=LstmMemoryModel
MODEL_DIR="../model/audio_lstmmemory1024_layer1_moe8"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="audio" \
--feature_sizes="128" \
--model=$MODEL \
--lstm_cells=512 \
--lstm_layers=1 \
--moe_num_mixtures=8 \
--rnn_swap_memory=False \
--batch_size=128 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstmmemory-layer1.sh
================================================
GPU_ID="1"
EVERY=2000
MODEL=LstmMemoryModel
MODEL_DIR="../model/lstmmemory1024_layer1_moe8"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--lstm_cells=1024 \
--lstm_layers=1 \
--moe_num_mixtures=8 \
--rnn_swap_memory=False \
--batch_size=128 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstmmemory.sh
================================================
GPU_ID="1"
EVERY=100
MODEL=LstmMemoryModel
MODEL_DIR="../model/lstmmemory_moe8_batch256"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=32 \
--model=$MODEL \
--moe_num_mixtures=8 \
--lstm_layers=2 \
--rnn_swap_memory=True \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-lstmoutput-parallel.sh
================================================
GPU_ID=1
EVERY=500
MODEL=LstmParallelFinaloutputModel
MODEL_DIR="../model/lstmparallelfinaloutput1024_moe8"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=64 \
--model=$MODEL\
--lstm_layers=2 \
--lstm_cells="1024,128" \
--moe_num_mixtures=8 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-mem.sh
================================================
GPU_ID=1
EVERY=500
MODEL=LstmMemoryModel
MODEL_DIR="../model/lstmmemory1024_moe8"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M-validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--model=$MODEL \
--lstm_cells=1024 \
--moe_num_mixtures=8 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-mm-lstm.sh
================================================
start=$1
GPU_ID=0
MODEL=MatchingMatrixLstmMemoryModel
MODEL_DIR="../model/mmlstmmemory1024_moe8"
for checkpoint in $(for filename in $MODEL_DIR/model.ckpt-*.meta; do echo $filename | grep -o "ckpt-[0123456789]*.meta" | cut -d '-' -f 2 | cut -d '.' -f 1; done | sort -n); do
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--model=$MODEL \
--mm_label_embedding=256 \
--moe_num_mixtures=8 \
--lstm_cells=1024 \
--lstm_layers=2 \
--rnn_swap_memory=True \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-moe-baseline.sh
================================================
GPU_ID=1
EVERY=100
MODEL=MoeModel
MODEL_DIR="../model/video_moe16_baseline"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--label_loss=CrossEntropyLoss \
--batch_size=128 \
--model=$MODEL \
--moe_num_mixtures=16 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-moe-batchagreement1.sh
================================================
GPU_ID=1
EVERY=100
MODEL=MoeModel
MODEL_DIR="../model/video_moe16_batchagreement0.001"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--label_loss=BatchAgreementCrossEntropyLoss \
--batch_agreement=0.001 \
--batch_size=128 \
--model=$MODEL \
--moe_num_mixtures=16 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-moe-batchagreement2.sh
================================================
GPU_ID=1
EVERY=100
MODEL=MoeModel
MODEL_DIR="../model/video_moe16_batchagreement0.003"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--label_loss=BatchAgreementCrossEntropyLoss \
--batch_agreement=0.003 \
--batch_size=128 \
--model=$MODEL \
--moe_num_mixtures=16 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-moe-batchagreement3.sh
================================================
GPU_ID=1
EVERY=100
MODEL=MoeModel
MODEL_DIR="../model/video_moe16_batchagreement0.01"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--label_loss=BatchAgreementCrossEntropyLoss \
--batch_agreement=0.01 \
--batch_size=128 \
--model=$MODEL \
--moe_num_mixtures=16 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-moe-model.sh
================================================
GPU_ID="1"
EVERY=100
MODEL=MoeModel
MODEL_DIR="../model/video_moe16_model"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--moe_num_mixtures=16 \
--batch_size=1024 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-moe-topk-batchagreement1.sh
================================================
GPU_ID=1
EVERY=100
MODEL=MoeModel
MODEL_DIR="../model/video_moe16_topk_batchagreement0.3"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--label_loss=TopKBatchAgreementCrossEntropyLoss \
--batch_agreement=0.3 \
--batch_size=128 \
--model=$MODEL \
--moe_num_mixtures=16 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-moe-topk-batchagreement2.sh
================================================
GPU_ID=1
EVERY=100
MODEL=MoeModel
MODEL_DIR="../model/video_moe16_topk_batchagreement1"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--label_loss=TopKBatchAgreementCrossEntropyLoss \
--batch_agreement=1 \
--batch_size=128 \
--model=$MODEL \
--moe_num_mixtures=16 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-moe-topk-batchagreement3.sh
================================================
GPU_ID=1
EVERY=100
MODEL=MoeModel
MODEL_DIR="../model/video_moe16_topk_batchagreement3"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--label_loss=TopKBatchAgreementCrossEntropyLoss \
--batch_agreement=3 \
--batch_size=128 \
--model=$MODEL \
--moe_num_mixtures=16 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-multi-lstmmem-deep-chain.sh
================================================
GPU_ID=1
EVERY=500
MODEL=LstmMemoryDeepChainModel
MODEL_DIR="../model/multilstmmemory1024_moe4_deep_chain"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=64 \
--model=$MODEL \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=200 \
--deep_chain_layers=1 \
--lstm_layers=2 \
--lstm_cells=1024 \
--multitask=True \
--support_type="label" \
--num_supports=4716 \
--label_loss=MultiTaskCrossEntropyLoss \
--support_loss_percent=0.2 \
--rnn_swap_memory=True \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-multi-view-cnn-deep-combine-chain.sh
================================================
GPU_ID=0
EVERY=1000
MODEL=MultiViewCnnDeepCombineChainModel
MODEL_DIR="../model/multi_view_cnn_deep_combine_chain"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--moe_num_mixtures=4 \
--deep_chain_layers=3 \
--deep_chain_relu_cells=64 \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label" \
--support_loss_percent=0.05 \
--batch_size=256 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-multires-lstm-deep-combine-chain.sh
================================================
GPU_ID=0
EVERY=1000
MODEL=MultiresLstmMemoryDeepCombineChainModel
MODEL_DIR="../model/multires_lstm_deep_combine_chain"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--lstm_cells="512,64" \
--deep_chain_layers=4 \
--deep_chain_relu_cells=256 \
--moe_num_mixtures=4 \
--label_loss=CrossEntropyLoss \
--dropout=True \
--batch_size=128 \
--feature_transformer=IdenticalTransformer \
--rnn_swap_memory=True \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-multitask-ce.sh
================================================
GPU_ID=1
EVERY=2000
MODEL=LstmMemoryMultitaskModel
MODEL_DIR="../model/lstmmemory1024_moe8_multitask_ce"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=32 \
--model=$MODEL \
--lstm_layers=2 \
--lstm_cells=1024 \
--multitask=True \
--moe_num_mixtures=8 \
--num_supports=25 \
--num_verticals=25 \
--vertical_file="resources/vertical.tsv" \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-multitask.sh
================================================
GPU_ID=0
EVERY=2000
MODEL=LstmMemoryMultitaskModel
MODEL_DIR="../model/lstmmemory1024_moe8_multitask"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=64 \
--model=$MODEL \
--lstm_layers=2 \
--lstm_cells=1024 \
--multitask=True \
--moe_num_mixtures=8 \
--num_supports=25 \
--num_verticals=25 \
--vertical_file="resources/vertical.tsv" \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-positional-cnn-dcc.sh
================================================
GPU_ID="0"
EVERY=1000
MODEL=PositionalCnnDeepCombineChainModel
MODEL_DIR="../model/positional_cnn_deep_combine_chain"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=64 \
--model=$MODEL \
--moe_num_mixtures=4 \
--deep_chain_layers=2 \
--deep_chain_relu_cells=128 \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label" \
--support_loss_percent=0.1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-stage2-logistic.sh
================================================
start=$1
GPU_ID=1
MODEL=Stage2LogisticModel
MODEL_DIR="../model/video_moe16_stage2logistic"
for checkpoint in $(for filename in $MODEL_DIR/model.ckpt-*.meta; do echo $filename | grep -o "ckpt-[0123456789]*.meta" | cut -d '-' -f 2 | cut -d '.' -f 1; done | sort -n); do
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate-validate-part1/validatea*" \
--frame_features=False \
--feature_names="predictions" \
--feature_sizes=4716 \
--batch_size=1024 \
--model=$MODEL \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-stage2-moe.sh
================================================
start=$1
GPU_ID=0
MODEL=MoeModel
MODEL_DIR="../model/video_moe16_stage2_moe8"
for checkpoint in $(for filename in $MODEL_DIR/model.ckpt-*.meta; do echo $filename | grep -o "ckpt-[0123456789]*.meta" | cut -d '-' -f 2 | cut -d '.' -f 1; done | sort -n); do
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate-validate-part1/validatea*" \
--frame_features=False \
--feature_names="predictions" \
--feature_sizes=4716 \
--batch_size=1024 \
--moe_num_mixtures=8 \
--model=$MODEL \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-video-deep-chain.sh
================================================
GPU_ID=1
EVERY=500
MODEL=DeepChainModel
MODEL_DIR="../model/video_deep_chain_moe4"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--model=$MODEL \
--deep_chain_relu_cells=200 \
--deep_chain_layers=3 \
--moe_num_mixtures=4 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-video-deep-combine-addnoise.sh
================================================
noise_level=$1
start=$2
GPU_ID=0
EVERY=100
MODEL=DeepCombineChainModel
MODEL_DIR="../model/video_chain_moe4_deep_combine_addnoise/after_relu_noise_level_$noise_level"
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label,label" \
--num_supports=18864 \
--support_loss_percent=0.05 \
--model=$MODEL \
--deep_chain_relu_cells=256 \
--deep_chain_layers=4 \
--moe_num_mixtures=4 \
--batch_size=128 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-video-deep-combine-chain-dropout.sh
================================================
GPU_ID=0
EVERY=100
MODEL=DeepCombineChainModel
MODEL_DIR="../model/video_chain_moe4_deep_combine_dropout/input_dropout_0.8"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--moe_num_mixtures=4 \
--deep_chain_layers=4 \
--deep_chain_relu_cells=256 \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label,label" \
--support_loss_percent=0.05 \
--batch_size=128 \
--model=$MODEL \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-video-deep-combine-chain-labelsmoothing.sh
================================================
GPU_ID=0
EVERY=100
MODEL=DeepCombineChainModel
MODEL_DIR="../model/video_chain_moe4_deep_combine_labelsmoothing"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--moe_num_mixtures=4 \
--deep_chain_layers=4 \
--deep_chain_relu_cells=256 \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label,label" \
--support_loss_percent=0.05 \
--batch_size=128 \
--model=$MODEL \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-video-deep-combine-chain-noise.sh
================================================
noise_level=$1
start=$2
GPU_ID=0
EVERY=100
MODEL=DeepCombineChainModel
MODEL_DIR="../model/video_deep_combine_chain_moe4_noise/input_noise_level_$noise_level"
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label,label" \
--num_supports=18864 \
--support_loss_percent=0.05 \
--model=$MODEL \
--deep_chain_relu_cells=256 \
--deep_chain_layers=4 \
--moe_num_mixtures=4 \
--batch_size=1024 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-video-deep-combine-chain.sh
================================================
GPU_ID=0
EVERY=100
MODEL=DeepCombineChainModel
MODEL_DIR="../model/video_chain_moe16_deep_combine"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--moe_num_mixtures=2 \
--deep_chain_layers=16 \
--deep_chain_relu_cells=256 \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label,label,label,label,label,label,label,label,label,label,label,label,label,label" \
--support_loss_percent=0.0125 \
--batch_size=512 \
--model=$MODEL \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-video-distillchain-video-dcc.sh
================================================
start=$1
GPU_ID=0
MODEL=DistillchainDeepCombineChainModel
MODEL_DIR="../model/distillchain_video_dcc"
for checkpoint in $(for filename in $MODEL_DIR/model.ckpt-*.meta; do echo $filename | grep -o "ckpt-[0123456789]*.meta" | cut -d '-' -f 2 | cut -d '.' -f 1; done | sort -n); do
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--distill_data_pattern="/Youtube-8M/model_predictions/validatea/distillation/ensemble_mean_model/*.tfrecord" \
--distillation_features=True \
--distillation_as_input=True \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--moe_num_mixtures=4 \
--deep_chain_layers=4 \
--deep_chain_relu_cells=256 \
--batch_size=256 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-video-divergence-chain-model.sh
================================================
GPU_ID=0
EVERY=200
MODEL=MultiTaskDivergenceDeepCombineChainModel
MODEL_DIR="../model/video_divergence_deep_combine_chain"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--model=$MODEL \
--lstm_cells="1024,128" \
--divergence_model_count=8 \
--deep_chain_layers=2 \
--deep_chain_relu_cells=128 \
--moe_num_mixtures=4 \
--multitask=True \
--label_loss=MultiTaskDivergenceCrossEntropyLoss \
--support_loss_percent=0.025 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-video-divergence-moe-model.sh
================================================
GPU_ID=1
EVERY=500
MODEL=MultiTaskDivergenceMoeModel
MODEL_DIR="../model/video_divergence_moe"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--model=$MODEL \
--model=MultiTaskDivergenceMoeModel \
--lstm_cells="1024,128" \
--divergence_model_count=8 \
--moe_num_mixtures=4 \
--multitask=True \
--label_loss=MultiTaskDivergenceCrossEntropyLoss \
--support_loss_percent=0.025 \
--num_readers=1 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-video-hidden-chain.sh
================================================
GPU_ID=0
EVERY=100
MODEL=HiddenChainModel
MODEL_DIR="../model/video_chain_moe16_hidden"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--multitask=True \
--support_type="label,label,label,label" \
--label_loss=MultiTaskCrossEntropyLoss \
--moe_num_mixtures=8 \
--batch_size=512 \
--model=$MODEL \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-video-hidden-combine-chain.sh
================================================
GPU_ID=0
EVERY=100
MODEL=HiddenCombineChainModel
MODEL_DIR="../model/video_chain_moe16_hidden_combine"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--moe_num_mixtures=8 \
--batch_size=512 \
--model=$MODEL \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-video-logistic.sh
================================================
GPU_ID=1
EVERY=100
MODEL=LogisticModel
MODEL_DIR="../model/video_logistic_model"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=$MODEL \
--batch_size=32 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-video-moe.sh
================================================
GPU_ID=1
EVERY=100
MODEL=MoeModel
MODEL_DIR="/home/zhangt/yt8m/video_level_moe_model"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=4096 \
--model=$MODEL \
--moe_num_mixtures=16 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-video-pairwise.sh
================================================
GPU_ID=1
EVERY=500
MODEL=MoeModel
MODEL_DIR="../model/video_moe16_mixedloss"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=256 \
--model=$MODEL \
--moe_num_mixtures=16 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_scripts/eval-video-verydeep-combine-chain.sh
================================================
GPU_ID=1
EVERY=100
MODEL=DeepCombineChainModel
MODEL_DIR="../model/video_chain_moe16_verydeep_combine"
start=$1
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [ $checkpoint -gt $start ]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--moe_num_mixtures=2 \
--deep_chain_layers=8 \
--deep_chain_relu_cells=128 \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label,label,label,label,label,label" \
--support_loss_percent=0.025 \
--batch_size=128 \
--model=$MODEL \
--run_once=True
fi
done
================================================
FILE: youtube-8m-wangheda/eval_util.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Provides functions to help with evaluating models."""
import datetime
import numpy
from tensorflow.python.platform import gfile
import mean_average_precision_calculator as map_calculator
import average_precision_calculator as ap_calculator
def flatten(l):
""" Merges a list of lists into a single list. """
return [item for sublist in l for item in sublist]
def calculate_hit_at_one(predictions, actuals):
"""Performs a local (numpy) calculation of the hit at one.
Args:
predictions: Matrix containing the outputs of the model.
Dimensions are 'batch' x 'num_classes'.
actuals: Matrix containing the ground truth labels.
Dimensions are 'batch' x 'num_classes'.
Returns:
float: The average hit at one across the entire batch.
"""
top_prediction = numpy.argmax(predictions, 1)
hits = actuals[numpy.arange(actuals.shape[0]), top_prediction]
return numpy.average(hits)
def calculate_recall_at_n(predictions, actuals, n):
"""Performs a local (numpy) calculation of the recall@n
Args:
predictions: Matrix containing the outputs of the model.
Dimensions are 'batch' x 'num_classes'.
actuals: Matrix containing the ground truth labels.
Dimensions are 'batch' x 'num_classes'.
n: scalar of n
Returns:
float: The recall at n across the entire batch.
"""
aggregated_recall = 0.0
num_videos = actuals.shape[0]
for row in numpy.arange(num_videos):
num_labels = int(numpy.sum(actuals[row]))
top_indices = numpy.argpartition(predictions[row],
-n)[-n:]
item_recall = 0.0
for label_index in top_indices:
if predictions[row][label_index] > 0:
item_recall += actuals[row][label_index]
item_recall /= num_labels
aggregated_recall += item_recall
aggregated_recall /= num_videos
return aggregated_recall
def calculate_precision_at_equal_recall_rate(predictions, actuals):
"""Performs a local (numpy) calculation of the PERR.
Args:
predictions: Matrix containing the outputs of the model.
Dimensions are 'batch' x 'num_classes'.
actuals: Matrix containing the ground truth labels.
Dimensions are 'batch' x 'num_classes'.
Returns:
float: The average precision at equal recall rate across the entire batch.
"""
aggregated_precision = 0.0
num_videos = actuals.shape[0]
for row in numpy.arange(num_videos):
num_labels = int(numpy.sum(actuals[row]))
top_indices = numpy.argpartition(predictions[row],
-num_labels)[-num_labels:]
item_precision = 0.0
for label_index in top_indices:
if predictions[row][label_index] > 0:
item_precision += actuals[row][label_index]
item_precision /= top_indices.size
aggregated_precision += item_precision
aggregated_precision /= num_videos
return aggregated_precision
def calculate_gap(predictions, actuals, top_k=20):
"""Performs a local (numpy) calculation of the global average precision.
Only the top_k predictions are taken for each of the videos.
Args:
predictions: Matrix containing the outputs of the model.
Dimensions are 'batch' x 'num_classes'.
actuals: Matrix containing the ground truth labels.
Dimensions are 'batch' x 'num_classes'.
top_k: How many predictions to use per video.
Returns:
float: The global average precision.
"""
gap_calculator = ap_calculator.AveragePrecisionCalculator()
sparse_predictions, sparse_labels, num_positives = top_k_by_class(predictions, actuals, top_k)
gap_calculator.accumulate(flatten(sparse_predictions), flatten(sparse_labels), sum(num_positives))
return gap_calculator.peek_ap_at_n()
def top_k_by_class(predictions, labels, k=20):
"""Extracts the top k predictions for each video, sorted by class.
Args:
predictions: A numpy matrix containing the outputs of the model.
Dimensions are 'batch' x 'num_classes'.
k: the top k non-zero entries to preserve in each prediction.
Returns:
A tuple (predictions,labels, true_positives). 'predictions' and 'labels'
are lists of lists of floats. 'true_positives' is a list of scalars. The
length of the lists are equal to the number of classes. The entries in the
predictions variable are probability predictions, and
the corresponding entries in the labels variable are the ground truth for
those predictions. The entries in 'true_positives' are the number of true
positives for each class in the ground truth.
Raises:
ValueError: An error occurred when the k is not a positive integer.
"""
if k <= 0:
raise ValueError("k must be a positive integer.")
k = min(k, predictions.shape[1])
num_classes = predictions.shape[1]
prediction_triplets= []
for video_index in range(predictions.shape[0]):
prediction_triplets.extend(top_k_triplets(predictions[video_index],labels[video_index], k))
out_predictions = [[] for v in range(num_classes)]
out_labels = [[] for v in range(num_classes)]
for triplet in prediction_triplets:
out_predictions[triplet[0]].append(triplet[1])
out_labels[triplet[0]].append(triplet[2])
out_true_positives = [numpy.sum(labels[:,i]) for i in range(num_classes)]
return out_predictions, out_labels, out_true_positives
def top_k_triplets(predictions, labels, k=20):
"""Get the top_k for a 1-d numpy array. Returns a sparse list of tuples in
(prediction, class) format"""
m = len(predictions)
k = min(k, m)
indices = numpy.argpartition(predictions, -k)[-k:]
return [(index, predictions[index], labels[index]) for index in indices]
class EvaluationMetrics(object):
"""A class to store the evaluation metrics."""
def __init__(self, num_class, top_k):
"""Construct an EvaluationMetrics object to store the evaluation metrics.
Args:
num_class: A positive integer specifying the number of classes.
top_k: A positive integer specifying how many predictions are considered per video.
Raises:
ValueError: An error occurred when MeanAveragePrecisionCalculator cannot
not be constructed.
"""
self.sum_hit_at_one = 0.0
self.sum_perr = 0.0
self.sum_loss = 0.0
self.map_calculator = map_calculator.MeanAveragePrecisionCalculator(num_class)
self.global_ap_calculator = ap_calculator.AveragePrecisionCalculator()
self.top_k = top_k
self.num_examples = 0
def accumulate(self, predictions, labels, loss):
"""Accumulate the metrics calculated locally for this mini-batch.
Args:
predictions: A numpy matrix containing the outputs of the model.
Dimensions are 'batch' x 'num_classes'.
labels: A numpy matrix containing the ground truth labels.
Dimensions are 'batch' x 'num_classes'.
loss: A numpy array containing the loss for each sample.
Returns:
dictionary: A dictionary storing the metrics for the mini-batch.
Raises:
ValueError: An error occurred when the shape of predictions and actuals
does not match.
"""
batch_size = labels.shape[0]
mean_hit_at_one = calculate_hit_at_one(predictions, labels)
mean_perr = calculate_precision_at_equal_recall_rate(predictions, labels)
mean_loss = numpy.mean(loss)
# Take the top 20 predictions.
sparse_predictions, sparse_labels, num_positives = top_k_by_class(predictions, labels, self.top_k)
self.map_calculator.accumulate(sparse_predictions, sparse_labels, num_positives)
self.global_ap_calculator.accumulate(flatten(sparse_predictions), flatten(sparse_labels), sum(num_positives))
self.num_examples += batch_size
self.sum_hit_at_one += mean_hit_at_one * batch_size
self.sum_perr += mean_perr * batch_size
self.sum_loss += mean_loss * batch_size
return {"hit_at_one": mean_hit_at_one, "perr": mean_perr, "loss": mean_loss}
def get(self):
"""Calculate the evaluation metrics for the whole epoch.
Raises:
ValueError: If no examples were accumulated.
Returns:
dictionary: a dictionary storing the evaluation metrics for the epoch. The
dictionary has the fields: avg_hit_at_one, avg_perr, avg_loss, and
aps (default nan).
"""
if self.num_examples <= 0:
raise ValueError("total_sample must be positive.")
avg_hit_at_one = self.sum_hit_at_one / self.num_examples
avg_perr = self.sum_perr / self.num_examples
avg_loss = self.sum_loss / self.num_examples
aps = self.map_calculator.peek_map_at_n()
gap = self.global_ap_calculator.peek_ap_at_n()
epoch_info_dict = {}
return {"avg_hit_at_one": avg_hit_at_one, "avg_perr": avg_perr,
"avg_loss": avg_loss, "aps": aps, "gap": gap}
def clear(self):
"""Clear the evaluation metrics and reset the EvaluationMetrics object."""
self.sum_hit_at_one = 0.0
self.sum_perr = 0.0
self.sum_loss = 0.0
self.map_calculator.clear()
self.global_ap_calculator.clear()
self.num_examples = 0
================================================
FILE: youtube-8m-wangheda/feature_transform.py
================================================
from tensorflow import flags
flags.DEFINE_string("feature_transformer", "DefaultTransformer",
"how to preprocess feature, defaults to identical, which means no transform")
flags.DEFINE_string("engineer_types", "identical,avg,std,diff",
"how to preprocess feature, defaults to identical, which means no transform")
flags.DEFINE_integer("time_resolution", 8,
"how many frames to mean pooling at a time")
from all_feature_transform import *
================================================
FILE: youtube-8m-wangheda/frame_level_models.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains a collection of models which operate on variable-length sequences.
"""
from tensorflow import flags
FLAGS = flags.FLAGS
flags.DEFINE_integer("iterations", 30,
"Number of frames per batch for DBoF.")
flags.DEFINE_bool("dbof_add_batch_norm", True,
"Adds batch normalization to the DBoF model.")
flags.DEFINE_bool(
"sample_random_frames", True,
"If true samples random frames (for frame level models). If false, a random"
"sequence of frames is sampled instead.")
flags.DEFINE_integer("dbof_cluster_size", 8192,
"Number of units in the DBoF cluster layer.")
flags.DEFINE_integer("dbof_hidden_size", 1024,
"Number of units in the DBoF hidden layer.")
flags.DEFINE_string("dbof_pooling_method", "max",
"The pooling method used in the DBoF cluster layer. "
"Choices are 'average' and 'max'.")
flags.DEFINE_string("video_level_classifier_model", "MoeModel",
"Some Frame-Level models can be decomposed into a "
"generalized pooling operation followed by a "
"classifier layer")
flags.DEFINE_string("video_level_classifier_support_model", "MoeModel",
"Some Frame-Level models can be decomposed into a "
"generalized pooling operation followed by a "
"classifier layer")
flags.DEFINE_bool("rnn_swap_memory", False, "If true, swap_memory = True.")
flags.DEFINE_string("lstm_cells", "1024", "Number of LSTM cells.")
flags.DEFINE_integer("lstm_layers", 2, "Number of LSTM layers.")
flags.DEFINE_integer("gru_cells", 1024, "Number of GRU cells.")
flags.DEFINE_integer("gru_layers", 2, "Number of GRU layers.")
flags.DEFINE_integer("attention_size", 1, "Number of attention layers.")
flags.DEFINE_integer("num_attentions", 5, "Number of attention cells per layer.")
flags.DEFINE_string("cnn_filter_sizes", "1,2,3", "Sizes of cnn filters.")
flags.DEFINE_string("cnn_filter_nums", "256,256,256", "Numbers of every cnn filters.")
flags.DEFINE_integer("cnn_pooling_k", 4, "The k value for max-k pooling.")
flags.DEFINE_string("lstm_normalization", "identical",
"which normalization method")
flags.DEFINE_integer("mm_label_embedding", 256,
"size of label embedding vector")
flags.DEFINE_string("wide_and_deep_models", "FrameLevelLogisticModel,LstmMemoryModel",
"size of label embedding vector")
flags.DEFINE_integer("deep_cnn_base_size", 128,
"basic cnn size")
flags.DEFINE_integer("lstm_look_back", 3,
"how many adjacent input for a cell to look at")
flags.DEFINE_integer("lstm_attentions", 8, "Attention size in lstm_attention_max_pooling_model.")
flags.DEFINE_integer("positional_embedding_size", 32, "Positional embedding dimension use in lstm_positional_attention_max_pooling_model.")
flags.DEFINE_bool("is_training", False, "used in batch normalization.")
flags.DEFINE_integer("multiscale_cnn_lstm_layers", 1, "number of layers in multiscale cnn_lstm.")
flags.DEFINE_integer("frame_seg_relu_cells", 256, "number of relu cells in frame-seg model.")
flags.DEFINE_integer("distillchain_relu_cells", 256, "number of relu cells in distillchain model.")
flags.DEFINE_integer("cnn_num_filters", 512, "number of filters in cnn conv layer.")
import sys
from os.path import dirname
if dirname(__file__) not in sys.path:
sys.path.append(dirname(__file__))
from all_frame_models import *
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-attentionlstm_moe4.sh
================================================
for part in test ensemble_train ensemble_validate train_samples; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/attentionlstm_moe4" \
--model_checkpoint_path="../model/attentionlstm_moe4/model.ckpt-104135" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=64 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-biunilstm1024_moe8.sh
================================================
for part in test ensemble_train ensemble_validate train_samples; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/biunilstm1024_moe4" \
--model_checkpoint_path="../model/biunilstm1024_moe4/model.ckpt-94882" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-cnn_deep_combine_chain.sh
================================================
#for part in ensemble_train ensemble_validate test; do
for part in train_samples; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/cnn_deep_combine_chain" \
--model_checkpoint_path="../model/cnn_deep_combine_chain/model.ckpt-228332" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-cnn_lstmmemory1024_moe8.sh
================================================
for part in test ensemble_train ensemble_validate train_samples; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/cnnlstmmemory1024_moe8" \
--model_checkpoint_path="../model/cnnlstmmemory1024_moe8/model.ckpt-149354" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=64 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-cnn_model.sh
================================================
#for part in ensemble_train ensemble_validate test; do
for part in ensemble_validate; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/cnn_model" \
--model_checkpoint_path="../model/cnn_model/model.ckpt-374098" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=CnnModel \
--cnn_num_filters=512 \
--moe_num_mixtures=4 \
--batch_size=128 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-dbof.sh
================================================
for part in test ensemble_train ensemble_validate; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/dbof_model" \
--model_checkpoint_path="../model/dbof_model/model.ckpt-184058" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=DbofModel \
--batch_size=64 \
--num_readers=1 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-deep_cnn_deep_combine.sh
================================================
#for part in ensemble_train ensemble_validate test; do
for part in train_samples; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/deep_cnn_deep_combine_chain" \
--model_checkpoint_path="../model/deep_cnn_deep_combine_chain/model.ckpt-303289" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-deeplstm1024_layer6_moe4.sh
================================================
for part in test ensemble_train ensemble_validate train_samples; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/deeplstm1024_layer6_moe4" \
--model_checkpoint_path="../model/deeplstm1024_layer6_moe4/model.ckpt-175048" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-distill_video_dcc.sh
================================================
for part in ensemble_train ensemble_validate test; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/distillation_video_dcc_noise" \
--model_checkpoint_path="../model/distillation_video_dcc_noise/scene1_percent_0.4/model.ckpt-47842" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=DeepCombineChainModel \
--deep_chain_relu_cells=256 \
--deep_chain_layers=4 \
--moe_num_mixtures=4 \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-distillation-cnn-dcc.sh
================================================
for part in train; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions_local/${part}/distillation/distillchain_cnn_dcc" \
--model_checkpoint_path="../model/distillation/distillchain_cnn_dcc/model.ckpt-74488" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_mean_model/*.tfrecord" \
--distillation_features=True \
--distillation_as_input=True \
--model=DistillchainCnnDeepCombineChainModel \
--deep_chain_layers=3 \
--deep_chain_relu_cells=256 \
--moe_num_mixtures=4 \
--batch_size=128 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-distillation-lstmattention8max.sh
================================================
# for distillation version 2 data
cd ../youtube-8m-zhangteng
#for part in ensemble_train ensemble_validate test; do
for part in train; do
output_dir="/Youtube-8M/model_predictions_x32/${part}/distillation/distillchain_lstm_attention8max"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble-distill.py \
--output_dir="$output_dir" \
--model_checkpoint_path="../model/distillation/frame_level_lstm_extend_distillchain_model/model.ckpt-74796" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--model=LstmExtendModel \
--video_level_classifier_model=MoeExtendDistillChainModel \
--moe_num_extend=8 \
--moe_num_mixtures=8 \
--train=False \
--batch_size=32 \
--file_size=4096
fi
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-distillation-lstmgate.sh
================================================
# for distillation version 2 data
cd ../youtube-8m-zhangteng
#for part in ensemble_train ensemble_validate test; do
for part in train; do
output_dir="/Youtube-8M/model_predictions_x32/${part}/distillation/distillchain_lstm_gate_moe8"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble-distill.py \
--output_dir="$output_dir" \
--model_checkpoint_path="../model/distillation/frame_level_lstm_gate_distillchain_model/model.ckpt-63344" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--model=LstmGateModel \
--video_level_classifier_model=MoeDistillChainModel \
--moe_num_extend=4 \
--moe_method=None \
--lstm_cells=1024 \
--moe_num_mixtures=8 \
--train=False \
--batch_size=32 \
--file_size=4096
fi
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-distillation-video-dcc.sh
================================================
#for part in ensemble_train ensemble_validate test; do
for part in train; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions_x32/${part}/distillation/distillchain_video_dcc" \
--model_checkpoint_path="../model/distillation/distillchain_video_dcc/model.ckpt-19296" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_mean_model/*.tfrecord" \
--distillation_features=True \
--distillation_as_input=True \
--model=DistillchainDeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_layers=4 \
--deep_chain_relu_cells=256 \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-distillation.sh
================================================
#!/bin/bash
# the script directory
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd ${DIR}/../
model_name="distillation/video_chain_moe2_verydeep_combine"
checkpoint="28403"
for part in ensemble_validate test ensemble_train; do
output_dir="/Youtube-8M/model_predictions/${part}/${model_name}"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="$output_dir" \
--model_checkpoint_path="../model/${model_name}/model.ckpt-${checkpoint}" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=DeepCombineChainModel \
--deep_chain_relu_cells=128 \
--deep_chain_layers=8 \
--moe_num_mixtures=2 \
--batch_size=1024 \
--file_size=4096
fi
done
model_name="distillation/cnn_deep_combine_chain"
checkpoint="228332"
for part in ensemble_validate test ensemble_train; do
output_dir="/Youtube-8M/model_predictions/${part}/${model_name}"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="$output_dir" \
--model_checkpoint_path="../model/${model_name}/model.ckpt-${checkpoint}" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=CnnDeepCombineChainModel \
--deep_chain_layers=4 \
--deep_chain_relu_cells=128 \
--moe_num_mixtures=4 \
--batch_size=128 \
--file_size=4096
fi
done
model_name="distillation/lstmparallelfinaloutput1024_moe8"
checkpoint="144351"
for part in ensemble_validate test ensemble_train; do
output_dir="/Youtube-8M/model_predictions/${part}/${model_name}"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="$output_dir" \
--model_checkpoint_path="../model/${model_name}/model.ckpt-${checkpoint}" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=LstmParallelFinaloutputModel \
--lstm_cells="1024,128" \
--moe_num_mixtures=8 \
--rnn_swap_memory=True \
--batch_size=128 \
--file_size=4096
fi
done
# The Lstm-Attention-Max-pooling model
cd ${DIR}/../../youtube-8m-zhangteng/
model_name="distillation/frame_level_lstm_extend8_model"
checkpoint="181785"
for part in ensemble_validate test ensemble_train; do
output_dir="/Youtube-8M/model_predictions/${part}/${model_name}"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES=0 python inference_with_rebuild.py \
--output_dir="$output_dir" \
--model_checkpoint_path="../model/${model_name}/model.ckpt-${checkpoint}" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=LstmExtendModel \
--video_level_classifier_model=MoeExtendModel \
--moe_num_mixtures=8 \
--moe_num_extend=8 \
--batch_size=128 \
--file_size=4096
fi
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-distillchain-cnn-dcc.sh
================================================
for part in ensemble_train test ensemble_validate; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/distillchain_cnn_dcc" \
--model_checkpoint_path="../model/distillchain_cnn_dcc/model.ckpt-113446" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_mean_model/*.tfrecord" \
--distillation_features=True \
--distillation_as_input=True \
--model=DistillchainCnnDeepCombineChainModel \
--deep_chain_layers=3 \
--deep_chain_relu_cells=256 \
--moe_num_mixtures=4 \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-distillchain-lstmcnn.sh
================================================
for part in ensemble_train ensemble_validate test; do
output_dir="/Youtube-8M/model_predictions/${part}/distillchain_lstmcnn_dcc"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble.py \
--output_dir="$output_dir" \
--model_checkpoint_path="../model/distillchain_lstmcnn_dcc/model.ckpt-79669" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_as_input=True \
--model=DistillchainLstmCnnDeepCombineChainModel \
--deep_chain_layers=2 \
--deep_chain_relu_cells=128 \
--distillchain_relu_cells=256 \
--moe_num_mixtures=4 \
--lstm_layers=1 \
--lstm_cells="1024,128" \
--rnn_swap_memory=True \
--batch_size=32 \
--file_size=4096
fi
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-distillchain-lstmparalleloutput.sh
================================================
for part in ensemble_train ensemble_validate test; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/distillchain_lstmparalleloutput" \
--model_checkpoint_path="../model/distillchain_lstmparalleloutput/model.ckpt-75261" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_mean_model/*.tfrecord" \
--distillation_features=True \
--distillation_as_input=True \
--model=DistillchainLstmParallelFinaloutputModel \
--rnn_swap_memory=True \
--lstm_cells="1024,128" \
--moe_num_mixtures=8 \
--batch_size=128 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-distillchain-v2-boost-lstmparalleloutput.sh
================================================
for part in ensemble_train ensemble_validate test; do
output_dir="/Youtube-8M/model_predictions/${part}/distillchain_v2_boost_lstmparalleloutput"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble.py \
--output_dir="$output_dir" \
--model_checkpoint_path="../model/distillchain_v2_boost_lstmparalleloutput/model.ckpt-86757" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_v2_matrix_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=False \
--distillation_as_input=True \
--model=DistillchainLstmParallelFinaloutputModel \
--rnn_swap_memory=True \
--lstm_layers=1 \
--lstm_cells="1024,128" \
--moe_num_mixtures=8 \
--batch_size=32 \
--file_size=4096
fi
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-distillchain-v2-lstmattention8max.sh
================================================
for part in ensemble_train ensemble_validate test; do
output_dir="/Youtube-8M/model_predictions/${part}/distillchain_v2_lstmattention8max"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="$output_dir" \
--model_checkpoint_path="../model/distillchain_v2_lstmattention8max/model.ckpt-" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_v2_matrix_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=False \
--distillation_as_input=True \
--model=DistillchainLstmAttentionMaxPoolingModel \
--moe_num_mixtures=8 \
--lstm_attentions=8 \
--lstm_cells=1024 \
--rnn_swap_memory=True \
--moe_num_mixtures=8 \
--batch_size=128 \
--file_size=4096
fi
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-distillchain-v2-lstmcnn.sh
================================================
for part in ensemble_train ensemble_validate test; do
output_dir="/Youtube-8M/model_predictions/${part}/distillchain_v2_lstmcnn_dcc"
if [ ! -d $output_dir ]; then
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="$output_dir" \
--model_checkpoint_path="../model/distillchain_v2_lstmcnn_dcc/model.ckpt-77277" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_v2_matrix_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=False \
--distillation_as_input=True \
--model=DistillchainLstmCnnDeepCombineChainModel \
--deep_chain_layers=2 \
--deep_chain_relu_cells=128 \
--distillchain_relu_cells=256 \
--moe_num_mixtures=4 \
--lstm_layers=1 \
--lstm_cells="1024,128" \
--rnn_swap_memory=True \
--batch_size=128 \
--file_size=4096
fi
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-distillchain-v2-lstmparalleloutput.sh
================================================
for part in ensemble_train ensemble_validate test; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/distillchain_v2_lstmparalleloutput" \
--model_checkpoint_path="../model/distillchain_v2_lstmparalleloutput/model.ckpt-74190" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_v2_matrix_model/*.tfrecord" \
--distillation_features=False \
--distillation_as_input=True \
--model=DistillchainLstmParallelFinaloutputModel \
--rnn_swap_memory=True \
--lstm_cells="1024,128" \
--moe_num_mixtures=4 \
--batch_size=128 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-distillchain-v2-multilstm.sh
================================================
for part in ensemble_train ensemble_validate test; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/distillchain_v2_multilstm_dcc" \
--model_checkpoint_path="../model/distillchain_v2_multilstm_dcc/model.ckpt-74272" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_v2_matrix_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=False \
--distillation_as_input=True \
--model=DistillchainLstmMemoryDeepCombineChainModel \
--lstm_layers=1 \
--lstm_cells=1024 \
--moe_num_mixtures=4 \
--distillation_relu_cells=256 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=2 \
--rnn_swap_memory=True \
--batch_size=128 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-distillchain-v2-multiscal-cnnlstm.sh
================================================
for part in ensemble_train ensemble_validate test; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/distillchain_v2_multiscale_cnnlstm" \
--model_checkpoint_path="../model/distillchain_v2_multiscale_cnnlstm/model.ckpt-72344" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_v2_matrix_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=False \
--distillation_as_input=True \
--model=DistillchainMultiscaleCnnLstmModel \
--multiscale_cnn_lstm_layers=3 \
--moe_num_mixtures=4 \
--rnn_swap_memory=True \
--is_training=False \
--batch_size=128 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-distillchain-v2-video-dcc.sh
================================================
for part in ensemble_train ensemble_validate test; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/distillchain_v2_video_dcc" \
--model_checkpoint_path="../model/distillchain_v2_video_dcc/model.ckpt-17137" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_v2_matrix_model/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_as_input=True \
--model=DistillchainDeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_layers=4 \
--deep_chain_relu_cells=256 \
--batch_size=1024 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-frame_seg.sh
================================================
for part in ensemble_train ensemble_validate test; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions_local/${part}/frame_seg_model" \
--model_checkpoint_path="../model/frame_seg_model/model.ckpt-27945" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--feature_transformer=IdenticalTransformer \
--model=FrameSegModel \
--moe_num_mixtures=16 \
--batch_size=64 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-framehop_lstm.sh
================================================
for part in test ensemble_train ensemble_validate train_samples; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/framehop_lstm" \
--model_checkpoint_path="../model/framehop_lstm/model.ckpt-143746" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--feature_transformer=IdenticalTransformer \
--batch_size=64 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-lstm_attention8_max.sh
================================================
#for part in ensemble_train ensemble_validate test; do
for part in train_samples; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/lstm_attention8_max" \
--model_checkpoint_path="/home/zhangt/yt8m/frame_level_lstm_extend8_model/model.ckpt-181785" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-lstm_cnn_deep_combine_chain.sh
================================================
for part in test ensemble_validate ensemble_train; do
#for part in train_samples; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/lstm_cnn_deep_combine_chain" \
--model_checkpoint_path="../model/lstm_cnn_deep_combine_chain/model.ckpt-259179" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-lstmattlstm1024_moe8.sh
================================================
for part in test ensemble_train ensemble_validate train_samples; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/lstmattlstm1024_moe8" \
--model_checkpoint_path="../model/lstmattlstm1024_moe8/model.ckpt-86809" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=64 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-lstmmemory-audio.sh
================================================
CUDA_VISIBLE_DEVICES=1 python inference.py \
--output_file="../model/audio_lstmmemory1024_layer1_moe8/error_analysis.train_samples.tsv" \
--model_checkpoint_path="../model/audio_lstmmemory1024_layer1_moe8/model.ckpt-187979" \
--input_data_pattern="/Youtube-8M/data/frame/train/train1r.tfrecord" \
--frame_features=True \
--feature_names="audio" \
--feature_sizes="128" \
--batch_size=128 \
--file_size=4096
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-lstmmemory-layer1.sh
================================================
#for part in ensemble_train ensemble_validate test; do
for part in ensemble_validate; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/lstmmemory1024_layer1_moe8" \
--model_checkpoint_path="../model/lstmmemory1024_layer1_moe8/model.ckpt-149022" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=LstmMemoryModel \
--lstm_cells=1024 \
--lstm_layers=1 \
--moe_num_mixtures=8 \
--rnn_swap_memory=False \
--batch_size=128 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-lstmmemory1024_deep_combine_chain_add_length.sh
================================================
#for part in ensemble_train ensemble_validate test; do
for part in train_samples; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/lstmmem1024_layer2_moe4_deep_combine_chain_add_length" \
--model_checkpoint_path="../model/lstmmem1024_deep_combine_chain_length/model.ckpt-148035" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-lstmmemory1024_moe8.sh
================================================
#for part in ensemble_train ensemble_validate test; do
for part in train_samples; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/lstmmemory_cell1024_layer2_moe8" \
--model_checkpoint_path="../model/lstmmemory1024_moe8/model.ckpt-123144" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-lstmmemory2048_moe4.sh
================================================
#for part in ensemble_train ensemble_validate test; do
for part in train_samples; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/lstmmemory_cell2048_layer2_moe4" \
--model_checkpoint_path="../model/lstmmemory2048_moe4_batch128/model.ckpt-109434" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-lstmparallelmemory1024_moe8.sh
================================================
#for part in ensemble_train ensemble_validate test; do
for part in train_samples; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/lstmparallelmemory1024_moe8" \
--model_checkpoint_path="../model/lstmparallelmemory1024_moe8/model.ckpt-111155" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-lstmparalleloutput1024_moe8.sh
================================================
for part in test ensemble_train ensemble_validate train_samples; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/lstmparallelfinaloutput1024_moe8" \
--model_checkpoint_path="../model/lstmparallelfinaloutput1024_moe8/model.ckpt-144351" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=64 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-model_input.sh
================================================
for part in test ensemble_train ensemble_validate; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble-get-input.py \
--output_dir="/Youtube-8M/model_predictions/${part}/model_input" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-multilstmmemory1024_moe4_deep_chain.sh
================================================
#for part in test ensemble_validate ensemble_train; do
for part in train_samples; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/multilstmmemory1024_moe4_deep_chain" \
--model_checkpoint_path="../model/multilstmmemory1024_moe4_deep_chain/model.ckpt-133508" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-multires_lstm_deep_combine_chain.sh
================================================
for part in test ensemble_validate ensemble_train; do
#for part in train_samples; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/multires_lstm_deep_combine_chain" \
--model_checkpoint_path="../model/multires_lstm_deep_combine_chain/model.ckpt-108969" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--feature_transformer=IdenticalTransformer \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-positional-lstmattention8max.sh
================================================
for part in test ensemble_train ensemble_validate; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/lstm_positional_attention8max" \
--model_checkpoint_path="../model/lstm_positional_attention8max/model.ckpt-198407" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=LstmPositionalAttentionMaxPoolingModel \
--moe_num_mixtures=8 \
--lstm_attentions=8 \
--positional_embedding_size=32 \
--rnn_swap_memory=True \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-video-distillchain-video-dcc.sh
================================================
for part in ensemble_train ensemble_validate test; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/distillchain_video_dcc" \
--model_checkpoint_path="../model/distillchain_video_dcc/model.ckpt-19296" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_mean_model/*.tfrecord" \
--distillation_features=True \
--distillation_as_input=True \
--model=DistillchainDeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_layers=4 \
--deep_chain_relu_cells=256 \
--batch_size=1024 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-video_group_moe4_noise0.2_layer4_elu.sh
================================================
#for part in ensemble_train ensemble_validate test; do
for part in train_samples; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/video_moe4_noise0.2_layer4_elu" \
--model_checkpoint_path="/home/zhangt/yt8m/video_level_moenoise0.2_model/moe_4layers_elu/model.ckpt-22845" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-video_logistic.sh
================================================
for part in ensemble_train ensemble_validate test; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/video_logistic_model" \
--model_checkpoint_path="../model/video_logistic_model/model.ckpt-23581" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=LogisticModel \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-video_moe16_model.sh
================================================
for part in ensemble_train ensemble_validate test; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/video_moe16_model" \
--model_checkpoint_path="../model/video_moe16_model/model.ckpt-19058" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=MoeModel \
--moe_num_mixtures=16 \
--batch_size=128 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/infer_scripts/infer-video_very_deep_combine_chain.sh
================================================
#for part in ensemble_train ensemble_validate test; do
for part in train_samples; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/video_very_deep_combine_chain" \
--model_checkpoint_path="../model/video_chain_moe2_verydeep_combine/model.ckpt-28403" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--file_size=4096
done
================================================
FILE: youtube-8m-wangheda/inference-layer.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for generating predictions over a set of videos."""
import os
import time
import numpy
import numpy as np
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import eval_util
import losses
import readers
import utils
FLAGS = flags.FLAGS
if __name__ == '__main__':
flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
"The directory to load the model files from.")
flags.DEFINE_string("model_checkpoint_path", "",
"The file path to load the model from.")
flags.DEFINE_string("output_file", "",
"The file to save the predictions to.")
flags.DEFINE_integer("file_size", 4096,
"Number of examples put into a file.")
flags.DEFINE_string(
"input_data_pattern", "",
"File glob defining the evaluation dataset in tensorflow.SequenceExample "
"format. The SequenceExamples are expected to have an 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
# Model flags.
flags.DEFINE_bool(
"frame_features", False,
"If set, then --eval_data_pattern must be frame-level features. "
"Otherwise, --eval_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_integer(
"batch_size", 8192,
"How many examples to process per batch.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024", "Length of the feature vectors.")
flags.DEFINE_string("layer_name", "model/RNN/concat:0", "Name of the layer"
"to extract value from.")
# Other flags.
flags.DEFINE_integer("num_readers", 1,
"How many threads to use for reading input files.")
flags.DEFINE_integer("top_k", 20,
"How many predictions to output per video.")
def format_lines(video_ids, predictions, top_k):
batch_size = len(video_ids)
for video_index in range(batch_size):
top_indices = numpy.argpartition(predictions[video_index], -top_k)[-top_k:]
line = [(class_index, predictions[video_index][class_index])
for class_index in top_indices]
# print("Type - Test :")
# print(type(video_ids[video_index]))
# print(video_ids[video_index].decode('utf-8'))
line = sorted(line, key=lambda p: -p[1])
yield video_ids[video_index].decode('utf-8') + "," + " ".join("%i %f" % pair
for pair in line) + "\n"
def get_input_data_tensors(reader, data_pattern, batch_size, num_readers=1):
"""Creates the section of the graph which reads the input data.
Args:
reader: A class which parses the input data.
data_pattern: A 'glob' style path to the data files.
batch_size: How many examples to process at a time.
num_readers: How many I/O threads to use.
Returns:
A tuple containing the features tensor, labels tensor, and optionally a
tensor containing the number of frames per video. The exact dimensions
depend on the reader being used.
Raises:
IOError: If no files matching the given pattern were found.
"""
with tf.name_scope("input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find input files. data_pattern='" +
data_pattern + "'")
logging.info("number of input files: " + str(len(files)))
filename_queue = tf.train.string_input_producer(
files, num_epochs=1, shuffle=False)
examples_and_labels = [reader.prepare_reader(filename_queue)
for _ in range(num_readers)]
video_id_batch, video_batch, video_label_batch, num_frames_batch = (
tf.train.batch_join(examples_and_labels,
batch_size=batch_size,
allow_smaller_final_batch = True,
enqueue_many=True))
return video_id_batch, video_batch, video_label_batch, num_frames_batch
def get_output_feature(video_id, labels, feature_dict):
feature_maps = {'video_id': tf.train.Feature(bytes_list=tf.train.BytesList(value=[video_id])),
'labels': tf.train.Feature(int64_list=tf.train.Int64List(value=labels))}
for feature_name in feature_dict:
feature_maps[feature_name] = tf.train.Feature(
float_list=tf.train.FloatList(value=feature_dict[feature_name]))
example = tf.train.Example(features=tf.train.Features(feature=feature_maps))
return example
def write_to_record(id_batch, label_batch, feature_dict, filenum, num_examples_processed):
writer = tf.python_io.TFRecordWriter(FLAGS.output_file+str(filenum)+'.tfrecord')
for i in range(num_examples_processed):
slice_dict = lambda d, j: dict([(k,d[k][j,:]) for k in d])
video_id = id_batch[i]
label = np.nonzero(label_batch[i,:])[0]
example = get_output_feature(video_id, label, slice_dict(feature_dict, i))
serialized = example.SerializeToString()
writer.write(serialized)
writer.close()
def inference(reader, train_dir, data_pattern, out_file_location, batch_size, top_k):
with tf.Session() as sess:
video_id_batch, video_batch, video_label_batch, num_frames_batch = get_input_data_tensors(reader, data_pattern, batch_size)
if FLAGS.model_checkpoint_path:
latest_checkpoint = FLAGS.model_checkpoint_path
else:
latest_checkpoint = tf.train.latest_checkpoint(train_dir)
if latest_checkpoint is None:
raise Exception("unable to find a checkpoint at location: %s" % train_dir)
else:
meta_graph_location = latest_checkpoint + ".meta"
logging.info("loading meta-graph: " + meta_graph_location)
saver = tf.train.import_meta_graph(meta_graph_location, clear_devices=True)
logging.info("restoring variables from " + latest_checkpoint)
saver.restore(sess, latest_checkpoint)
input_tensor = tf.get_collection("input_batch_raw")[0]
num_frames_tensor = tf.get_collection("num_frames")[0]
layer_tensor = tf.get_default_graph().get_tensor_by_name(FLAGS.layer_name)
print "layer", layer_tensor
# Workaround for num_epochs issue.
def set_up_init_ops(variables):
init_op_list = []
for variable in list(variables):
if "train_input" in variable.name:
init_op_list.append(tf.assign(variable, 1))
variables.remove(variable)
init_op_list.append(tf.variables_initializer(variables))
return init_op_list
sess.run(set_up_init_ops(tf.get_collection_ref(
tf.GraphKeys.LOCAL_VARIABLES)))
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
num_examples_processed = 0
start_time = time.time()
try:
file_num = 0
video_id_list = []
video_label_list = []
video_layer_list = []
def write_record(video_id_list, video_label_list, video_layer_list, file_num, num_examples_processed):
video_id_list = np.concatenate(video_id_list,axis=0)
video_label_list = np.concatenate(video_label_list,axis=0)
video_layer_list = np.concatenate(video_layer_list,axis=0)
feature_dict = {"layer": video_layer_list}
write_to_record(video_id_list, video_label_list, feature_dict, file_num, num_examples_processed)
while not coord.should_stop():
video_id_batch_val, video_batch_val, video_label_batch_val, num_frames_batch_val = sess.run([video_id_batch, video_batch, video_label_batch, num_frames_batch])
layer_val = sess.run(layer_tensor, feed_dict={input_tensor: video_batch_val, num_frames_tensor: num_frames_batch_val})
now = time.time()
num_examples_processed += len(video_batch_val)
video_id_list.append(video_id_batch_val)
video_label_list.append(video_label_batch_val)
video_layer_list.append(layer_val)
if num_examples_processed>=FLAGS.file_size:
write_record(video_id_list, video_label_list, video_layer_list, file_num, num_examples_processed)
file_num += 1
video_id_list = []
video_label_list = []
video_layer_list = []
num_examples_processed = 0
logging.info("num examples processed: " + str(num_examples_processed) + " elapsed seconds: " + "{0:.2f}".format(now-start_time))
if num_examples_processed>0:
write_record(video_id_list, video_label_list, video_layer_list, file_num, num_examples_processed)
except tf.errors.OutOfRangeError:
logging.info('Done with inference. The output file was written to ' + out_file_location)
finally:
coord.request_stop()
coord.join(threads)
sess.close()
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if FLAGS.output_file is "":
raise ValueError("'output_file' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
inference(reader, FLAGS.train_dir, FLAGS.input_data_pattern,
FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-wangheda/inference-pre-ensemble-get-input.py
================================================
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# WARNING: This is a temporary script for writing the input directly into files, do not edit base on the file
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for generating predictions over a set of videos."""
import os
import time
import numpy
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import eval_util
import losses
import readers
import utils
import numpy as np
FLAGS = flags.FLAGS
if __name__ == '__main__':
flags.DEFINE_string("model_checkpoint_path", "",
"The file path to load the model from.")
flags.DEFINE_string("output_dir", "",
"The file to save the predictions to.")
flags.DEFINE_string(
"input_data_pattern", "",
"File glob defining the evaluation dataset in tensorflow.SequenceExample "
"format. The SequenceExamples are expected to have an 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
# Model flags.
flags.DEFINE_bool(
"frame_features", False,
"If set, then --eval_data_pattern must be frame-level features. "
"Otherwise, --eval_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_integer(
"batch_size", 8192,
"How many examples to process per batch.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024", "Length of the feature vectors.")
flags.DEFINE_integer("file_size", 4096,
"Number of frames per batch for DBoF.")
# Other flags.
flags.DEFINE_integer("num_readers", 1,
"How many threads to use for reading input files.")
flags.DEFINE_integer("top_k", 20,
"How many predictions to output per video.")
def get_input_data_tensors(reader, data_pattern, batch_size, num_readers=1):
"""Creates the section of the graph which reads the input data.
Args:
reader: A class which parses the input data.
data_pattern: A 'glob' style path to the data files.
batch_size: How many examples to process at a time.
num_readers: How many I/O threads to use.
Returns:
A tuple containing the features tensor, labels tensor, and optionally a
tensor containing the number of frames per video. The exact dimensions
depend on the reader being used.
Raises:
IOError: If no files matching the given pattern were found.
"""
with tf.name_scope("input"):
files = gfile.Glob(data_pattern)
files.sort()
if not files:
raise IOError("Unable to find input files. data_pattern='" +
data_pattern + "'")
logging.info("number of input files: " + str(len(files)))
filename_queue = tf.train.string_input_producer(
files, num_epochs=1, shuffle=False)
examples_and_labels = [reader.prepare_reader(filename_queue)
for _ in range(num_readers)]
video_id_batch, video_batch, unused_labels, num_frames_batch = (
tf.train.batch_join(examples_and_labels,
batch_size=batch_size,
allow_smaller_final_batch=True,
enqueue_many=True))
return video_id_batch, video_batch, unused_labels, num_frames_batch
def inference(reader, model_checkpoint_path, data_pattern, out_file_location, batch_size, top_k):
with tf.Session() as sess:
video_id_batch, video_batch, video_label_batch, num_frames_batch = get_input_data_tensors(reader, data_pattern, batch_size)
# Workaround for num_epochs issue.
def set_up_init_ops(variables):
init_op_list = []
for variable in list(variables):
if "train_input" in variable.name:
init_op_list.append(tf.assign(variable, 1))
variables.remove(variable)
init_op_list.append(tf.variables_initializer(variables))
return init_op_list
sess.run(set_up_init_ops(tf.get_collection_ref(
tf.GraphKeys.LOCAL_VARIABLES)))
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
num_examples_processed = 0
start_time = time.time()
video_id = []
video_label = []
video_inputs = []
video_features = []
filenum = 0
directory = FLAGS.output_dir
if not os.path.exists(directory):
os.makedirs(directory)
else:
raise IOError("Output path exists! path='" + directory + "'")
try:
while not coord.should_stop():
video_id_batch_val, video_batch_val, video_label_batch_val, num_frames_batch_val = sess.run([video_id_batch, video_batch, video_label_batch, num_frames_batch])
now = time.time()
num_examples_processed += len(video_batch_val)
video_id.append(video_id_batch_val)
video_label.append(video_label_batch_val)
video_inputs.append(video_batch_val)
if num_examples_processed>=FLAGS.file_size:
assert num_examples_processed==FLAGS.file_size, "num_examples_processed should be equal to file_size"
video_id = np.concatenate(video_id,axis=0)
video_label = np.concatenate(video_label,axis=0)
video_inputs = np.concatenate(video_inputs,axis=0)
write_to_record(video_id, video_label, video_inputs, filenum, num_examples_processed)
filenum += 1
video_id = []
video_label = []
video_inputs = []
num_examples_processed = 0
logging.info("num examples processed: " + str(num_examples_processed) + " elapsed seconds: " + "{0:.2f}".format(now-start_time))
except tf.errors.OutOfRangeError:
logging.info('Done with inference. The output file was written to ' + out_file_location)
finally:
coord.request_stop()
if num_examples_processedij", all_distill_labels, distill_weight)
feature_transformer = transformer_class()
model_input, num_frames = feature_transformer.transform(model_input_raw, num_frames=num_frames)
with tf.name_scope("model"):
if FLAGS.noise_level > 0:
noise_level_tensor = tf.placeholder_with_default(0.0, shape=[], name="noise_level")
else:
noise_level_tensor = None
if distill_readers is not None:
distillation_predictions = final_distill_labels
else:
distillation_predictions = None
if FLAGS.dropout:
keep_prob_tensor = tf.placeholder_with_default(1.0, shape=[], name="keep_prob")
result = model.create_model(
model_input,
num_frames=num_frames,
vocab_size=reader.num_classes,
labels=labels_batch,
dropout=FLAGS.dropout,
keep_prob=keep_prob_tensor,
noise_level=noise_level_tensor,
distillation_predictions=distillation_predictions,
is_training=False)
else:
result = model.create_model(
model_input,
num_frames=num_frames,
vocab_size=reader.num_classes,
labels=labels_batch,
noise_level=noise_level_tensor,
distillation_predictions=distillation_predictions,
is_training=False)
print "result", result
predictions = result["predictions"]
tf.add_to_collection("predictions", predictions)
tf.add_to_collection("video_id_batch", video_id)
tf.add_to_collection("input_batch_raw", model_input_raw)
tf.add_to_collection("input_batch", model_input)
tf.add_to_collection("num_frames", num_frames)
tf.add_to_collection("labels", tf.cast(labels_batch, tf.float32))
if FLAGS.dropout:
tf.add_to_collection("keep_prob", keep_prob_tensor)
if FLAGS.noise_level > 0:
tf.add_to_collection("noise_level", noise_level_tensor)
def inference(saver, model_checkpoint_path, out_file_location, batch_size, top_k):
with tf.Session() as sess:
print model_checkpoint_path, FLAGS.train_dir
if model_checkpoint_path is None:
model_checkpoint_path = tf.train.latest_checkpoint(FLAGS.train_dir)
print model_checkpoint_path, FLAGS.train_dir
if model_checkpoint_path is None:
raise Exception("unable to find a checkpoint at location: %s" % model_checkpoint_path)
logging.info("restoring variables from " + model_checkpoint_path)
saver.restore(sess, model_checkpoint_path)
input_tensor = tf.get_collection("input_batch_raw")[0]
num_frames_tensor = tf.get_collection("num_frames")[0]
predictions_tensor = tf.get_collection("predictions")[0]
video_id_tensor = tf.get_collection("video_id_batch")[0]
labels_tensor = tf.get_collection("labels")[0]
init_op = tf.global_variables_initializer()
# Workaround for num_epochs issue.
def set_up_init_ops(variables):
init_op_list = []
for variable in list(variables):
if "train_input" in variable.name:
init_op_list.append(tf.assign(variable, 1))
variables.remove(variable)
init_op_list.append(tf.variables_initializer(variables))
return init_op_list
sess.run(set_up_init_ops(tf.get_collection_ref(
tf.GraphKeys.LOCAL_VARIABLES)))
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
start_time = time.time()
filenum = 0
video_id = []
video_label = []
video_features = []
num_examples_processed = 0
directory = FLAGS.output_dir
if not os.path.exists(directory):
os.makedirs(directory)
else:
raise IOError("Output path exists! path='" + directory + "'")
try:
while not coord.should_stop():
predictions_batch_val, video_id_batch_val, labels_batch_val = sess.run([predictions_tensor, video_id_tensor, labels_tensor])
video_id.append(video_id_batch_val)
video_label.append(labels_batch_val)
video_features.append(predictions_batch_val)
num_examples_processed += len(video_id_batch_val)
now = time.time()
logging.info("num examples processed: " + str(num_examples_processed) + " elapsed seconds: " + "{0:.2f}".format(now-start_time))
if num_examples_processed >= FLAGS.file_size:
assert num_examples_processed==FLAGS.file_size, "num_examples_processed should be equal to file_size"
video_id = np.concatenate(video_id, axis=0)
video_label = np.concatenate(video_label, axis=0)
video_features = np.concatenate(video_features, axis=0)
write_to_record(video_id, video_label, video_features, filenum, num_examples_processed)
filenum += 1
video_id = []
video_label = []
video_features = []
num_examples_processed = 0
except tf.errors.OutOfRangeError:
logging.info('Done with inference. The output file was written to ' + out_file_location)
finally:
coord.request_stop()
if 0 < num_examples_processed <= FLAGS.file_size:
video_id = np.concatenate(video_id,axis=0)
video_label = np.concatenate(video_label,axis=0)
video_features = np.concatenate(video_features,axis=0)
write_to_record(video_id, video_label, video_features, filenum,num_examples_processed)
coord.join(threads)
sess.close()
def write_to_record(id_batch, label_batch, predictions, filenum, num_examples_processed):
writer = tf.python_io.TFRecordWriter(FLAGS.output_dir + '/' + 'predictions-%04d.tfrecord' % filenum)
for i in range(num_examples_processed):
video_id = id_batch[i]
label = np.nonzero(label_batch[i,:])[0]
example = get_output_feature(video_id, label, [predictions[i,:]], ['predictions'])
serialized = example.SerializeToString()
writer.write(serialized)
writer.close()
def get_output_feature(video_id, labels, features, feature_names):
feature_maps = {'video_id': tf.train.Feature(bytes_list=tf.train.BytesList(value=[video_id])),
'labels': tf.train.Feature(int64_list=tf.train.Int64List(value=labels))}
for feature_index in range(len(feature_names)):
feature_maps[feature_names[feature_index]] = tf.train.Feature(
float_list=tf.train.FloatList(value=features[feature_index]))
example = tf.train.Example(features=tf.train.Features(feature=feature_maps))
return example
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if FLAGS.output_dir is "":
raise ValueError("'output_dir' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
if FLAGS.distill_data_pattern is not None:
predictions_patterns = FLAGS.distill_data_pattern.strip().split(",")
predictions_readers = []
for pattern in predictions_patterns:
predictions_readers.append(
readers.YT8MAggregatedFeatureReader(
feature_names=["predictions"],
feature_sizes=[4716]))
else:
predictions_patterns = None
predictions_readers = None
model = find_class_by_name(FLAGS.model,
[frame_level_models, video_level_models])()
transformer_fn = find_class_by_name(FLAGS.feature_transformer,
[feature_transform])
build_graph(reader=reader,
input_data_pattern=FLAGS.input_data_pattern,
model=model,
distill_readers=predictions_readers,
distill_data_patterns=predictions_patterns,
batch_size=FLAGS.batch_size,
transformer_class=transformer_fn)
saver = tf.train.Saver(max_to_keep=3, keep_checkpoint_every_n_hours=10000000000)
inference(saver, FLAGS.model_checkpoint_path,
FLAGS.output_dir, FLAGS.batch_size, FLAGS.top_k)
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-wangheda/inference-pre-ensemble.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for generating predictions over a set of videos."""
import gc
import os
import time
import eval_util
import losses
import frame_level_models
import video_level_models
import data_augmentation
import feature_transform
import readers
import utils
import numpy
import numpy as np
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
FLAGS = flags.FLAGS
if __name__ == '__main__':
flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
"The directory to load the model files from.")
flags.DEFINE_string("model_checkpoint_path", None,
"The file path to load the model from.")
flags.DEFINE_string("output_dir", "",
"The file to save the predictions to.")
flags.DEFINE_string(
"input_data_pattern", "",
"File glob defining the evaluation dataset in tensorflow.SequenceExample "
"format. The SequenceExamples are expected to have an 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
flags.DEFINE_string(
"distill_data_pattern", None,
"File glob defining the distillation data pattern")
# Model flags.
flags.DEFINE_bool(
"frame_features", False,
"If set, then --eval_data_pattern must be frame-level features. "
"Otherwise, --eval_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_integer(
"batch_size", 8192,
"How many examples to process per batch.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024", "Length of the feature vectors.")
flags.DEFINE_integer("file_size", 4096,
"Number of frames per batch for DBoF.")
flags.DEFINE_string(
"model", "YouShouldSpecifyAModel",
"Which architecture to use for the model. Models are defined "
"in models.py.")
# Other flags.
flags.DEFINE_integer("num_readers", 1,
"How many threads to use for reading input files.")
flags.DEFINE_integer("top_k", 20,
"How many predictions to output per video.")
flags.DEFINE_bool(
"dropout", False,
"Whether to consider dropout")
flags.DEFINE_float("keep_prob", 1.0,
"probability to keep output (used in dropout, keep it unchanged in validationg and test)")
flags.DEFINE_float("noise_level", 0.0,
"standard deviation of noise (added to hidden nodes)")
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def get_input_data_tensors(reader, data_pattern, batch_size, num_readers=1):
"""Creates the section of the graph which reads the input data.
Args:
reader: A class which parses the input data.
data_pattern: A 'glob' style path to the data files.
batch_size: How many examples to process at a time.
num_readers: How many I/O threads to use.
Returns:
A tuple containing the features tensor, labels tensor, and optionally a
tensor containing the number of frames per video. The exact dimensions
depend on the reader being used.
Raises:
IOError: If no files matching the given pattern were found.
"""
with tf.name_scope("input"):
files = gfile.Glob(data_pattern)
files.sort()
if not files:
raise IOError("Unable to find input files. data_pattern='" +
data_pattern + "'")
logging.info("number of input files: " + str(len(files)))
filename_queue = tf.train.string_input_producer(
files, num_epochs=1, shuffle=False)
examples_and_labels = reader.prepare_reader(filename_queue)
video_id_batch, video_batch, unused_labels, num_frames_batch = (
tf.train.batch(examples_and_labels,
batch_size=batch_size,
capacity=batch_size * 8,
allow_smaller_final_batch=True,
enqueue_many=True))
return video_id_batch, video_batch, unused_labels, num_frames_batch
def build_graph(reader,
model,
input_data_pattern,
label_loss_fn=losses.CrossEntropyLoss(),
batch_size=1000,
distill_reader=None,
transformer_class=feature_transform.DefaultTransformer):
video_id, model_input_raw, labels_batch, num_frames = (
get_input_data_tensors(
reader,
input_data_pattern,
batch_size=batch_size))
if distill_reader is not None:
unused_video_id_batch, distill_input_raw, unused_labels_batch, unused_num_frames = get_input_data_tensors( # pylint: disable=g-line-too-long
distill_reader,
FLAGS.distill_data_pattern,
batch_size=batch_size)
feature_transformer = transformer_class()
model_input, num_frames = feature_transformer.transform(model_input_raw, num_frames=num_frames)
with tf.name_scope("model"):
if FLAGS.noise_level > 0:
noise_level_tensor = tf.placeholder_with_default(0.0, shape=[], name="noise_level")
else:
noise_level_tensor = None
if distill_reader is not None:
distillation_predictions = distill_input_raw
else:
distillation_predictions = None
if FLAGS.dropout:
keep_prob_tensor = tf.placeholder_with_default(1.0, shape=[], name="keep_prob")
result = model.create_model(
model_input,
num_frames=num_frames,
vocab_size=reader.num_classes,
labels=labels_batch,
dropout=FLAGS.dropout,
keep_prob=keep_prob_tensor,
noise_level=noise_level_tensor,
distillation_predictions=distillation_predictions,
is_training=False)
else:
result = model.create_model(
model_input,
num_frames=num_frames,
vocab_size=reader.num_classes,
labels=labels_batch,
noise_level=noise_level_tensor,
distillation_predictions=distillation_predictions,
is_training=False)
print "result", result
predictions = result["predictions"]
tf.add_to_collection("predictions", predictions)
tf.add_to_collection("video_id_batch", video_id)
tf.add_to_collection("input_batch_raw", model_input_raw)
tf.add_to_collection("input_batch", model_input)
tf.add_to_collection("num_frames", num_frames)
tf.add_to_collection("labels", tf.cast(labels_batch, tf.float32))
if FLAGS.dropout:
tf.add_to_collection("keep_prob", keep_prob_tensor)
if FLAGS.noise_level > 0:
tf.add_to_collection("noise_level", noise_level_tensor)
def inference(saver, model_checkpoint_path, out_file_location, batch_size, top_k):
with tf.Session() as sess:
print model_checkpoint_path, FLAGS.train_dir
if model_checkpoint_path is None:
model_checkpoint_path = tf.train.latest_checkpoint(FLAGS.train_dir)
print model_checkpoint_path, FLAGS.train_dir
if model_checkpoint_path is None:
raise Exception("unable to find a checkpoint at location: %s" % model_checkpoint_path)
logging.info("restoring variables from " + model_checkpoint_path)
saver.restore(sess, model_checkpoint_path)
input_tensor = tf.get_collection("input_batch_raw")[0]
num_frames_tensor = tf.get_collection("num_frames")[0]
predictions_tensor = tf.get_collection("predictions")[0]
video_id_tensor = tf.get_collection("video_id_batch")[0]
labels_tensor = tf.get_collection("labels")[0]
init_op = tf.global_variables_initializer()
# Workaround for num_epochs issue.
def set_up_init_ops(variables):
init_op_list = []
for variable in list(variables):
if "train_input" in variable.name:
init_op_list.append(tf.assign(variable, 1))
variables.remove(variable)
init_op_list.append(tf.variables_initializer(variables))
return init_op_list
sess.run(set_up_init_ops(tf.get_collection_ref(
tf.GraphKeys.LOCAL_VARIABLES)))
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
start_time = time.time()
filenum = 0
video_id = []
video_label = []
video_features = []
num_examples_processed = 0
directory = FLAGS.output_dir
if not os.path.exists(directory):
os.makedirs(directory)
else:
raise IOError("Output path exists! path='" + directory + "'")
try:
while not coord.should_stop():
predictions_batch_val, video_id_batch_val, labels_batch_val = sess.run([predictions_tensor, video_id_tensor, labels_tensor])
video_id.append(video_id_batch_val)
video_label.append(labels_batch_val)
video_features.append(predictions_batch_val)
num_examples_processed += len(video_id_batch_val)
now = time.time()
logging.info("num examples processed: " + str(num_examples_processed) + " elapsed seconds: " + "{0:.2f}".format(now-start_time))
if num_examples_processed >= FLAGS.file_size:
assert num_examples_processed==FLAGS.file_size, "num_examples_processed should be equal to file_size"
video_id = np.concatenate(video_id, axis=0)
video_label = np.concatenate(video_label, axis=0)
video_features = np.concatenate(video_features, axis=0)
write_to_record(video_id, video_label, video_features, filenum, num_examples_processed)
filenum += 1
video_id = []
video_label = []
video_features = []
num_examples_processed = 0
except tf.errors.OutOfRangeError:
logging.info('Done with inference. The output file was written to ' + out_file_location)
finally:
coord.request_stop()
if 0 < num_examples_processed <= FLAGS.file_size:
video_id = np.concatenate(video_id,axis=0)
video_label = np.concatenate(video_label,axis=0)
video_features = np.concatenate(video_features,axis=0)
write_to_record(video_id, video_label, video_features, filenum,num_examples_processed)
coord.join(threads)
sess.close()
def write_to_record(id_batch, label_batch, predictions, filenum, num_examples_processed):
writer = tf.python_io.TFRecordWriter(FLAGS.output_dir + '/' + 'predictions-%04d.tfrecord' % filenum)
for i in range(num_examples_processed):
video_id = id_batch[i]
label = np.nonzero(label_batch[i,:])[0]
example = get_output_feature(video_id, label, [predictions[i,:]], ['predictions'])
serialized = example.SerializeToString()
writer.write(serialized)
writer.close()
def get_output_feature(video_id, labels, features, feature_names):
feature_maps = {'video_id': tf.train.Feature(bytes_list=tf.train.BytesList(value=[video_id])),
'labels': tf.train.Feature(int64_list=tf.train.Int64List(value=labels))}
for feature_index in range(len(feature_names)):
feature_maps[feature_names[feature_index]] = tf.train.Feature(
float_list=tf.train.FloatList(value=features[feature_index]))
example = tf.train.Example(features=tf.train.Features(feature=feature_maps))
return example
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if FLAGS.output_dir is "":
raise ValueError("'output_dir' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
if FLAGS.distill_data_pattern is not None:
distill_reader = readers.YT8MAggregatedFeatureReader(feature_names=["predictions"],
feature_sizes=[4716])
else:
distill_reader = None
model = find_class_by_name(FLAGS.model,
[frame_level_models, video_level_models])()
transformer_fn = find_class_by_name(FLAGS.feature_transformer,
[feature_transform])
build_graph(reader,
model,
input_data_pattern=FLAGS.input_data_pattern,
batch_size=FLAGS.batch_size,
distill_reader=distill_reader,
transformer_class=transformer_fn)
saver = tf.train.Saver(max_to_keep=3, keep_checkpoint_every_n_hours=10000000000)
inference(saver, FLAGS.model_checkpoint_path,
FLAGS.output_dir, FLAGS.batch_size, FLAGS.top_k)
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-wangheda/inference-sample-error-analysis.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for generating error rates over a set of videos."""
import os
import time
import numpy
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import eval_util
import losses
import readers
import utils
FLAGS = flags.FLAGS
if __name__ == '__main__':
flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
"The directory to load the model files from.")
flags.DEFINE_string("model_checkpoint_path", None,
"The file path to load the model from.")
flags.DEFINE_string("output_file", "",
"The file to save the predictions to.")
flags.DEFINE_string(
"input_data_pattern", "",
"File glob defining the evaluation dataset in tensorflow.SequenceExample "
"format. The SequenceExamples are expected to have an 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
# Model flags.
flags.DEFINE_bool(
"frame_features", False,
"If set, then --eval_data_pattern must be frame-level features. "
"Otherwise, --eval_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_integer(
"batch_size", 8192,
"How many examples to process per batch.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024", "Length of the feature vectors.")
# Other flags.
flags.DEFINE_integer("num_readers", 1,
"How many threads to use for reading input files.")
flags.DEFINE_integer("top_k", 20,
"How many predictions to output per video.")
flags.DEFINE_bool(
"dropout", False,
"Whether to consider dropout")
flags.DEFINE_float("keep_prob", 1.0,
"probability to keep output (used in dropout, keep it unchanged in validationg and test)")
def format_lines(video_ids, predictions, labels, top_k):
batch_size = len(video_ids)
for video_index in range(batch_size):
n_recall = max(int(numpy.sum(labels[video_index])), 1)
# labels
label_indices = numpy.argpartition(labels[video_index], -n_recall)[-n_recall:]
label_predictions = [(class_index, predictions[video_index][class_index])
for class_index in label_indices]
label_predictions = sorted(label_predictions, key=lambda p: -p[1])
label_str = "\t".join(["%d\t%f"%(x,y) for x,y in label_predictions])
# predictions
top_k_indices = numpy.argpartition(predictions[video_index], -top_k)[-top_k:]
top_k_predictions = [(class_index, predictions[video_index][class_index])
for class_index in top_k_indices]
top_k_predictions = sorted(top_k_predictions, key=lambda p: -p[1])
top_k_str = "\t".join(["%d\t%f"%(x,y) for x,y in top_k_predictions])
# compute PERR
top_n_indices = numpy.argpartition(predictions[video_index], -n_recall)[-n_recall:]
positives = [labels[video_index][class_index]
for class_index in top_n_indices]
perr = sum(positives) / float(n_recall)
# URL
url = "https://www.youtube.com/watch?v=" + video_ids[video_index].decode('utf-8')
yield url + "\t" + str(1-perr) + "\t" + top_k_str + "\t" + label_str + "\n"
def get_input_data_tensors(reader, data_pattern, batch_size, num_readers=1):
"""Creates the section of the graph which reads the input data.
Args:
reader: A class which parses the input data.
data_pattern: A 'glob' style path to the data files.
batch_size: How many examples to process at a time.
num_readers: How many I/O threads to use.
Returns:
A tuple containing the features tensor, labels tensor, and optionally a
tensor containing the number of frames per video. The exact dimensions
depend on the reader being used.
Raises:
IOError: If no files matching the given pattern were found.
"""
with tf.name_scope("input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find input files. data_pattern='" +
data_pattern + "'")
logging.info("number of input files: " + str(len(files)))
filename_queue = tf.train.string_input_producer(
files, num_epochs=1, shuffle=False)
examples_and_labels = [reader.prepare_reader(filename_queue)
for _ in range(num_readers)]
video_id_batch, video_batch, labels_batch, num_frames_batch = (
tf.train.batch_join(examples_and_labels,
batch_size=batch_size,
allow_smaller_final_batch = True,
enqueue_many=True))
return video_id_batch, video_batch, labels_batch, num_frames_batch
def inference(reader, train_dir, data_pattern, out_file_location, batch_size, top_k):
with tf.Session() as sess, gfile.Open(out_file_location, "w+") as out_file:
video_id_batch, video_batch, labels_batch, num_frames_batch = get_input_data_tensors(reader, data_pattern, batch_size)
if FLAGS.model_checkpoint_path:
latest_checkpoint = FLAGS.model_checkpoint_path
else:
latest_checkpoint = tf.train.latest_checkpoint(train_dir)
if latest_checkpoint is None:
raise Exception("unable to find a checkpoint at location: %s" % train_dir)
else:
meta_graph_location = latest_checkpoint + ".meta"
logging.info("loading meta-graph: " + meta_graph_location)
saver = tf.train.import_meta_graph(meta_graph_location, clear_devices=True)
logging.info("restoring variables from " + latest_checkpoint)
saver.restore(sess, latest_checkpoint)
input_tensor = tf.get_collection("input_batch_raw")[0]
num_frames_tensor = tf.get_collection("num_frames")[0]
predictions_tensor = tf.get_collection("predictions")[0]
if FLAGS.dropout:
keep_prob_tensor = tf.get_collection("keep_prob")[0]
# Workaround for num_epochs issue.
def set_up_init_ops(variables):
init_op_list = []
for variable in list(variables):
if "train_input" in variable.name:
init_op_list.append(tf.assign(variable, 1))
variables.remove(variable)
init_op_list.append(tf.variables_initializer(variables))
return init_op_list
sess.run(set_up_init_ops(tf.get_collection_ref(
tf.GraphKeys.LOCAL_VARIABLES)))
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
num_examples_processed = 0
start_time = time.time()
out_file.write("VideoId,LabelConfidencePairs\n")
try:
while not coord.should_stop():
video_id_batch_val, video_batch_val, labels_batch_val, num_frames_batch_val = sess.run([video_id_batch, video_batch, labels_batch, num_frames_batch])
if FLAGS.dropout:
predictions_val, = sess.run([predictions_tensor], feed_dict={input_tensor: video_batch_val, num_frames_tensor: num_frames_batch_val, keep_prob_tensor: FLAGS.keep_prob})
else:
predictions_val, = sess.run([predictions_tensor], feed_dict={input_tensor: video_batch_val, num_frames_tensor: num_frames_batch_val})
now = time.time()
num_examples_processed += len(video_batch_val)
num_classes = predictions_val.shape[1]
logging.info("num examples processed: " + str(num_examples_processed) + " elapsed seconds: " + "{0:.2f}".format(now-start_time))
for line in format_lines(video_id_batch_val, predictions_val, labels_batch_val, top_k):
out_file.write(line)
out_file.flush()
except tf.errors.OutOfRangeError:
logging.info('Done with inference. The output file was written to ' + out_file_location)
finally:
coord.request_stop()
coord.join(threads)
sess.close()
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if FLAGS.output_file is "":
raise ValueError("'output_file' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
inference(reader, FLAGS.train_dir, FLAGS.input_data_pattern,
FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-wangheda/inference-sample-error.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for generating error rates over a set of videos."""
import os
import gc
import time
import eval_util
import losses
import frame_level_models
import video_level_models
import data_augmentation
import feature_transform
import readers
import utils
import numpy
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
FLAGS = flags.FLAGS
if __name__ == '__main__':
flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
"The directory to load the model files from.")
flags.DEFINE_string("model_checkpoint_path", None,
"The file path to load the model from.")
flags.DEFINE_string("output_file", "",
"The file to save the predictions to.")
flags.DEFINE_string(
"input_data_pattern", "",
"File glob defining the evaluation dataset in tensorflow.SequenceExample "
"format. The SequenceExamples are expected to have an 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
# Model flags.
flags.DEFINE_bool(
"frame_features", False,
"If set, then --eval_data_pattern must be frame-level features. "
"Otherwise, --eval_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_integer(
"batch_size", 8192,
"How many examples to process per batch.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024", "Length of the feature vectors.")
flags.DEFINE_string(
"model", "YouShouldSpecifyAModel",
"Which architecture to use for the model. Models are defined "
"in models.py.")
# Other flags.
flags.DEFINE_integer("num_readers", 2,
"How many threads to use for reading input files.")
flags.DEFINE_integer("top_k", 20,
"How many predictions to output per video.")
flags.DEFINE_bool(
"dropout", False,
"Whether to consider dropout")
flags.DEFINE_float("keep_prob", 1.0,
"probability to keep output (used in dropout, keep it unchanged in validationg and test)")
flags.DEFINE_float("noise_level", 0.0,
"standard deviation of noise (added to hidden nodes)")
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def format_lines(video_ids, predictions, labels):
batch_size = len(video_ids)
for video_index in range(batch_size):
# error rate
top_k = max(int(numpy.sum(labels[video_index])), 1)
top_indices = numpy.argpartition(predictions[video_index], -top_k)[-top_k:]
positives = [labels[video_index][class_index] for class_index in top_indices]
perr = sum(positives) / float(top_k)
yield video_ids[video_index].decode('utf-8') + "\t" + str(1-perr) + "\n"
def get_input_data_tensors(reader, data_pattern, batch_size, num_readers=1):
"""Creates the section of the graph which reads the input data.
Args:
reader: A class which parses the input data.
data_pattern: A 'glob' style path to the data files.
batch_size: How many examples to process at a time.
num_readers: How many I/O threads to use.
Returns:
A tuple containing the features tensor, labels tensor, and optionally a
tensor containing the number of frames per video. The exact dimensions
depend on the reader being used.
Raises:
IOError: If no files matching the given pattern were found.
"""
with tf.name_scope("input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find input files. data_pattern='" +
data_pattern + "'")
logging.info("number of input files: " + str(len(files)))
filename_queue = tf.train.string_input_producer(
files, num_epochs=1, shuffle=False)
examples_and_labels = [reader.prepare_reader(filename_queue)
for _ in range(num_readers)]
video_id_batch, video_batch, labels_batch, num_frames_batch = (
tf.train.batch_join(examples_and_labels,
batch_size=batch_size,
capacity=batch_size * 3,
allow_smaller_final_batch = True,
enqueue_many=True))
return video_id_batch, video_batch, labels_batch, num_frames_batch
def build_graph(reader,
model,
input_data_pattern,
label_loss_fn=losses.CrossEntropyLoss(),
batch_size=1000,
transformer_class=feature_transform.DefaultTransformer):
video_id, model_input_raw, labels_batch, num_frames = (
get_input_data_tensors(
reader,
input_data_pattern,
batch_size=batch_size,
num_readers=FLAGS.num_readers))
feature_transformer = transformer_class()
model_input, num_frames = feature_transformer.transform(model_input_raw, num_frames=num_frames)
with tf.name_scope("model"):
if FLAGS.noise_level > 0:
noise_level_tensor = tf.placeholder_with_default(0.0, shape=[], name="noise_level")
else:
noise_level_tensor = None
if FLAGS.dropout:
keep_prob_tensor = tf.placeholder_with_default(1.0, shape=[], name="keep_prob")
result = model.create_model(
model_input,
num_frames=num_frames,
vocab_size=reader.num_classes,
labels=labels_batch,
dropout=FLAGS.dropout,
keep_prob=keep_prob_tensor,
noise_level=noise_level_tensor)
else:
result = model.create_model(
model_input,
num_frames=num_frames,
vocab_size=reader.num_classes,
labels=labels_batch,
noise_level=noise_level_tensor)
print "result", result
predictions = result["predictions"]
tf.add_to_collection("predictions", predictions)
tf.add_to_collection("video_id_batch", video_id)
tf.add_to_collection("input_batch_raw", model_input_raw)
tf.add_to_collection("input_batch", model_input)
tf.add_to_collection("num_frames", num_frames)
tf.add_to_collection("labels", tf.cast(labels_batch, tf.float32))
if FLAGS.dropout:
tf.add_to_collection("keep_prob", keep_prob_tensor)
if FLAGS.noise_level > 0:
tf.add_to_collection("noise_level", noise_level_tensor)
def inference(saver, train_dir, out_file_location, batch_size, top_k):
with tf.Session() as sess, open(out_file_location, "w") as out_file:
if FLAGS.model_checkpoint_path:
latest_checkpoint = FLAGS.model_checkpoint_path
else:
latest_checkpoint = tf.train.latest_checkpoint(train_dir)
if latest_checkpoint is None:
raise Exception("unable to find a checkpoint at location: %s" % train_dir)
logging.info("restoring variables from " + latest_checkpoint)
saver.restore(sess, latest_checkpoint)
input_tensor = tf.get_collection("input_batch_raw")[0]
num_frames_tensor = tf.get_collection("num_frames")[0]
predictions_tensor = tf.get_collection("predictions")[0]
video_id_tensor = tf.get_collection("video_id_batch")[0]
labels_tensor = tf.get_collection("labels")[0]
init_op = tf.global_variables_initializer()
# Workaround for num_epochs issue.
def set_up_init_ops(variables):
init_op_list = []
for variable in list(variables):
if "train_input" in variable.name:
init_op_list.append(tf.assign(variable, 1))
variables.remove(variable)
init_op_list.append(tf.variables_initializer(variables))
return init_op_list
sess.run(set_up_init_ops(tf.get_collection_ref(
tf.GraphKeys.LOCAL_VARIABLES)))
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
num_examples_processed = 0
start_time = time.time()
out_file.write("VideoId,LabelConfidencePairs\n")
try:
while not coord.should_stop():
# for speed reason, here we use the graph, instead of using feed_dict and numpy arrays
predictions_val, video_id_batch_val, labels_batch_val = sess.run([predictions_tensor, video_id_tensor, labels_tensor])
num_examples_processed += len(video_id_batch_val)
num_classes = predictions_val.shape[1]
out_file.writelines([line for line in format_lines(video_id_batch_val, predictions_val, labels_batch_val)])
now = time.time()
logging.info("num examples processed: " + str(num_examples_processed) + " elapsed seconds: " + "{0:.2f}".format(now-start_time))
except tf.errors.OutOfRangeError:
logging.info('Done with inference. The output file was written to ' + out_file_location)
finally:
coord.request_stop()
out_file.flush()
coord.join(threads)
sess.close()
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if FLAGS.output_file is "":
raise ValueError("'output_file' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
model = find_class_by_name(FLAGS.model,
[frame_level_models, video_level_models])()
transformer_fn = find_class_by_name(FLAGS.feature_transformer,
[feature_transform])
build_graph(reader,
model,
input_data_pattern=FLAGS.input_data_pattern,
batch_size=FLAGS.batch_size,
transformer_class=transformer_fn)
saver = tf.train.Saver(max_to_keep=3, keep_checkpoint_every_n_hours=10000000000)
inference(saver, FLAGS.train_dir,
FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-wangheda/inference-stage1.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for generating predictions over a set of videos."""
import os
import time
import numpy
import numpy as np
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import eval_util
import losses
import readers
import utils
FLAGS = flags.FLAGS
if __name__ == '__main__':
flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
"The directory to load the model files from.")
flags.DEFINE_string("model_checkpoint_path", "",
"The file path to load the model from.")
flags.DEFINE_string("output_file", "",
"The file to save the predictions to.")
flags.DEFINE_integer("file_size", 4096,
"Number of examples put into a file.")
flags.DEFINE_string(
"input_data_pattern", "",
"File glob defining the evaluation dataset in tensorflow.SequenceExample "
"format. The SequenceExamples are expected to have an 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
# Model flags.
flags.DEFINE_bool(
"frame_features", False,
"If set, then --eval_data_pattern must be frame-level features. "
"Otherwise, --eval_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_integer(
"batch_size", 8192,
"How many examples to process per batch.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024", "Length of the feature vectors.")
# Other flags.
flags.DEFINE_integer("num_readers", 1,
"How many threads to use for reading input files.")
flags.DEFINE_integer("top_k", 20,
"How many predictions to output per video.")
def format_lines(video_ids, predictions, top_k):
batch_size = len(video_ids)
for video_index in range(batch_size):
top_indices = numpy.argpartition(predictions[video_index], -top_k)[-top_k:]
line = [(class_index, predictions[video_index][class_index])
for class_index in top_indices]
# print("Type - Test :")
# print(type(video_ids[video_index]))
# print(video_ids[video_index].decode('utf-8'))
line = sorted(line, key=lambda p: -p[1])
yield video_ids[video_index].decode('utf-8') + "," + " ".join("%i %f" % pair
for pair in line) + "\n"
def get_input_data_tensors(reader, data_pattern, batch_size, num_readers=1):
"""Creates the section of the graph which reads the input data.
Args:
reader: A class which parses the input data.
data_pattern: A 'glob' style path to the data files.
batch_size: How many examples to process at a time.
num_readers: How many I/O threads to use.
Returns:
A tuple containing the features tensor, labels tensor, and optionally a
tensor containing the number of frames per video. The exact dimensions
depend on the reader being used.
Raises:
IOError: If no files matching the given pattern were found.
"""
with tf.name_scope("input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find input files. data_pattern='" +
data_pattern + "'")
logging.info("number of input files: " + str(len(files)))
filename_queue = tf.train.string_input_producer(
files, num_epochs=1, shuffle=False)
examples_and_labels = [reader.prepare_reader(filename_queue)
for _ in range(num_readers)]
video_id_batch, video_batch, video_label_batch, num_frames_batch = (
tf.train.batch_join(examples_and_labels,
batch_size=batch_size,
allow_smaller_final_batch = True,
enqueue_many=True))
return video_id_batch, video_batch, video_label_batch, num_frames_batch
def get_output_feature(video_id, labels, feature_dict):
feature_maps = {'video_id': tf.train.Feature(bytes_list=tf.train.BytesList(value=[video_id])),
'labels': tf.train.Feature(int64_list=tf.train.Int64List(value=labels))}
for feature_name in feature_dict:
feature_maps[feature_name] = tf.train.Feature(
float_list=tf.train.FloatList(value=feature_dict[feature_name]))
example = tf.train.Example(features=tf.train.Features(feature=feature_maps))
return example
def write_to_record(id_batch, label_batch, feature_dict, filenum, num_examples_processed):
writer = tf.python_io.TFRecordWriter(FLAGS.output_file+str(filenum)+'.tfrecord')
for i in range(num_examples_processed):
slice_dict = lambda d, j: dict([(k,d[k][j,:]) for k in d])
video_id = id_batch[i]
label = np.nonzero(label_batch[i,:])[0]
example = get_output_feature(video_id, label, slice_dict(feature_dict, i))
serialized = example.SerializeToString()
writer.write(serialized)
writer.close()
def inference(reader, train_dir, data_pattern, out_file_location, batch_size, top_k):
with tf.Session() as sess:
video_id_batch, video_batch, video_label_batch, num_frames_batch = get_input_data_tensors(reader, data_pattern, batch_size)
if FLAGS.model_checkpoint_path:
latest_checkpoint = FLAGS.model_checkpoint_path
else:
latest_checkpoint = tf.train.latest_checkpoint(train_dir)
if latest_checkpoint is None:
raise Exception("unable to find a checkpoint at location: %s" % train_dir)
else:
meta_graph_location = latest_checkpoint + ".meta"
logging.info("loading meta-graph: " + meta_graph_location)
saver = tf.train.import_meta_graph(meta_graph_location, clear_devices=True)
logging.info("restoring variables from " + latest_checkpoint)
saver.restore(sess, latest_checkpoint)
input_tensor = tf.get_collection("input_batch_raw")[0]
num_frames_tensor = tf.get_collection("num_frames")[0]
predictions_tensor = tf.get_collection("predictions")[0]
# Workaround for num_epochs issue.
def set_up_init_ops(variables):
init_op_list = []
for variable in list(variables):
if "train_input" in variable.name:
init_op_list.append(tf.assign(variable, 1))
variables.remove(variable)
init_op_list.append(tf.variables_initializer(variables))
return init_op_list
sess.run(set_up_init_ops(tf.get_collection_ref(
tf.GraphKeys.LOCAL_VARIABLES)))
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
num_examples_processed = 0
start_time = time.time()
try:
file_num = 0
video_id_list = []
video_label_list = []
video_predictions_list = []
def write_record(video_id_list, video_label_list, video_predictions_list, file_num, num_examples_processed):
video_id_list = np.concatenate(video_id_list,axis=0)
video_label_list = np.concatenate(video_label_list,axis=0)
video_predictions_list = np.concatenate(video_predictions_list,axis=0)
feature_dict = {"predictions": video_predictions_list}
write_to_record(video_id_list, video_label_list, feature_dict, file_num, num_examples_processed)
while not coord.should_stop():
video_id_batch_val, video_batch_val, video_label_batch_val, num_frames_batch_val = sess.run([video_id_batch, video_batch, video_label_batch, num_frames_batch])
predictions_val = sess.run(predictions_tensor, feed_dict={input_tensor: video_batch_val, num_frames_tensor: num_frames_batch_val})
now = time.time()
num_examples_processed += len(video_batch_val)
video_id_list.append(video_id_batch_val)
video_label_list.append(video_label_batch_val)
video_predictions_list.append(predictions_val)
if num_examples_processed>=FLAGS.file_size:
write_record(video_id_list, video_label_list, video_predictions_list, file_num, num_examples_processed)
file_num += 1
video_id_list = []
video_label_list = []
video_predictions_list = []
num_examples_processed = 0
logging.info("num examples processed: " + str(num_examples_processed) + " elapsed seconds: " + "{0:.2f}".format(now-start_time))
if num_examples_processed>0:
write_record(video_id_list, video_label_list, video_predictions_list, file_num, num_examples_processed)
except tf.errors.OutOfRangeError:
logging.info('Done with inference. The output file was written to ' + out_file_location)
finally:
coord.request_stop()
coord.join(threads)
sess.close()
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if FLAGS.output_file is "":
raise ValueError("'output_file' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
inference(reader, FLAGS.train_dir, FLAGS.input_data_pattern,
FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-wangheda/inference.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for generating predictions over a set of videos."""
import os
import time
import numpy
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import eval_util
import losses
import readers
import utils
FLAGS = flags.FLAGS
if __name__ == '__main__':
flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
"The directory to load the model files from.")
flags.DEFINE_string("model_checkpoint_path", "",
"The file path to load the model from.")
flags.DEFINE_string("output_file", "",
"The file to save the predictions to.")
flags.DEFINE_string(
"input_data_pattern", "",
"File glob defining the evaluation dataset in tensorflow.SequenceExample "
"format. The SequenceExamples are expected to have an 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
# Model flags.
flags.DEFINE_bool(
"frame_features", False,
"If set, then --eval_data_pattern must be frame-level features. "
"Otherwise, --eval_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_integer(
"batch_size", 8192,
"How many examples to process per batch.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024", "Length of the feature vectors.")
# Other flags.
flags.DEFINE_integer("num_readers", 1,
"How many threads to use for reading input files.")
flags.DEFINE_integer("top_k", 20,
"How many predictions to output per video.")
flags.DEFINE_bool(
"dropout", False,
"Whether to consider dropout")
flags.DEFINE_float("keep_prob", 1.0,
"probability to keep output (used in dropout, keep it unchanged in validationg and test)")
def format_lines(video_ids, predictions, top_k):
batch_size = len(video_ids)
for video_index in range(batch_size):
top_indices = numpy.argpartition(predictions[video_index], -top_k)[-top_k:]
line = [(class_index, predictions[video_index][class_index])
for class_index in top_indices]
# print("Type - Test :")
# print(type(video_ids[video_index]))
# print(video_ids[video_index].decode('utf-8'))
line = sorted(line, key=lambda p: -p[1])
yield video_ids[video_index].decode('utf-8') + "," + " ".join("%i %f" % pair
for pair in line) + "\n"
def get_input_data_tensors(reader, data_pattern, batch_size, num_readers=1):
"""Creates the section of the graph which reads the input data.
Args:
reader: A class which parses the input data.
data_pattern: A 'glob' style path to the data files.
batch_size: How many examples to process at a time.
num_readers: How many I/O threads to use.
Returns:
A tuple containing the features tensor, labels tensor, and optionally a
tensor containing the number of frames per video. The exact dimensions
depend on the reader being used.
Raises:
IOError: If no files matching the given pattern were found.
"""
with tf.name_scope("input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find input files. data_pattern='" +
data_pattern + "'")
logging.info("number of input files: " + str(len(files)))
filename_queue = tf.train.string_input_producer(
files, num_epochs=1, shuffle=False)
examples_and_labels = [reader.prepare_reader(filename_queue)
for _ in range(num_readers)]
video_id_batch, video_batch, unused_labels, num_frames_batch = (
tf.train.batch_join(examples_and_labels,
batch_size=batch_size,
allow_smaller_final_batch = True,
enqueue_many=True))
return video_id_batch, video_batch, num_frames_batch
def inference(reader, train_dir, data_pattern, out_file_location, batch_size, top_k):
with tf.Session() as sess, gfile.Open(out_file_location, "w+") as out_file:
video_id_batch, video_batch, num_frames_batch = get_input_data_tensors(reader, data_pattern, batch_size)
if FLAGS.model_checkpoint_path:
latest_checkpoint = FLAGS.model_checkpoint_path
else:
latest_checkpoint = tf.train.latest_checkpoint(train_dir)
if latest_checkpoint is None:
raise Exception("unable to find a checkpoint at location: %s" % train_dir)
else:
meta_graph_location = latest_checkpoint + ".meta"
logging.info("loading meta-graph: " + meta_graph_location)
saver = tf.train.import_meta_graph(meta_graph_location, clear_devices=True)
logging.info("restoring variables from " + latest_checkpoint)
saver.restore(sess, latest_checkpoint)
input_tensor = tf.get_collection("input_batch_raw")[0]
num_frames_tensor = tf.get_collection("num_frames")[0]
predictions_tensor = tf.get_collection("predictions")[0]
if FLAGS.dropout:
keep_prob_tensor = tf.get_collection("keep_prob")[0]
# Workaround for num_epochs issue.
def set_up_init_ops(variables):
init_op_list = []
for variable in list(variables):
if "train_input" in variable.name:
init_op_list.append(tf.assign(variable, 1))
variables.remove(variable)
init_op_list.append(tf.variables_initializer(variables))
return init_op_list
sess.run(set_up_init_ops(tf.get_collection_ref(
tf.GraphKeys.LOCAL_VARIABLES)))
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
num_examples_processed = 0
start_time = time.time()
out_file.write("VideoId,LabelConfidencePairs\n")
try:
while not coord.should_stop():
video_id_batch_val, video_batch_val,num_frames_batch_val = sess.run([video_id_batch, video_batch, num_frames_batch])
if FLAGS.dropout:
predictions_val, = sess.run([predictions_tensor], feed_dict={input_tensor: video_batch_val, num_frames_tensor: num_frames_batch_val, keep_prob_tensor: FLAGS.keep_prob})
else:
predictions_val, = sess.run([predictions_tensor], feed_dict={input_tensor: video_batch_val, num_frames_tensor: num_frames_batch_val})
now = time.time()
num_examples_processed += len(video_batch_val)
num_classes = predictions_val.shape[1]
logging.info("num examples processed: " + str(num_examples_processed) + " elapsed seconds: " + "{0:.2f}".format(now-start_time))
for line in format_lines(video_id_batch_val, predictions_val, top_k):
out_file.write(line)
out_file.flush()
except tf.errors.OutOfRangeError:
logging.info('Done with inference. The output file was written to ' + out_file_location)
finally:
coord.request_stop()
coord.join(threads)
sess.close()
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if FLAGS.output_file is "":
raise ValueError("'output_file' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
inference(reader, FLAGS.train_dir, FLAGS.input_data_pattern,
FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-wangheda/losses.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Provides definitions for non-regularized training or test losses."""
import numpy as np
import tensorflow as tf
from tensorflow import flags
FLAGS = flags.FLAGS
flags.DEFINE_float("false_negative_punishment", 1.0,
"punishment constant to 1 classified to 0")
flags.DEFINE_float("false_positive_punishment", 1.0,
"punishment constant to 0 classified to 1")
flags.DEFINE_integer("num_classes", 4716,
"number of classes")
flags.DEFINE_float("support_loss_percent", 0.1,
"the part that support loss (in multi-task scenario) take in the whole loss function.")
flags.DEFINE_string("support_type", "vertical",
"type of support label, vertical or frequent or vertical,frequent.")
flags.DEFINE_integer("num_supports", 25, "Number of total support categories.")
flags.DEFINE_integer("num_verticals", 25, "Number of total vertical categories.")
flags.DEFINE_integer("num_frequents", 200, "Number of total frequent categories.")
flags.DEFINE_string("vertical_file", "resources/vertical.tsv", "Location of label-vertical mapping file.")
flags.DEFINE_float("batch_agreement", 0.1,
"the batch_agreement parameter")
flags.DEFINE_bool("label_smoothing", False,
"whether do label smoothing")
flags.DEFINE_float("label_smoothing_epsilon", 0.1,
"whether do label smoothing")
def smoothing(labels):
print "label smoothing for", labels
epsilon = FLAGS.label_smoothing_epsilon
float_labels = tf.cast(labels, tf.float32)
num_labels = tf.reduce_sum(float_labels, axis=1, keep_dims=True)
K = float_labels.get_shape().as_list()[1]
prior = num_labels / K
smooth_labels = float_labels * (1.0 - epsilon) + prior * epsilon
return smooth_labels
class BaseLoss(object):
"""Inherit from this class when implementing new losses."""
def calculate_loss(self, unused_predictions, unused_labels, **unused_params):
"""Calculates the average loss of the examples in a mini-batch.
Args:
unused_predictions: a 2-d tensor storing the prediction scores, in which
each row represents a sample in the mini-batch and each column
represents a class.
unused_labels: a 2-d tensor storing the labels, which has the same shape
as the unused_predictions. The labels must be in the range of 0 and 1.
unused_params: loss specific parameters.
Returns:
A scalar loss tensor.
"""
raise NotImplementedError()
class WeightedCrossEntropyLoss(BaseLoss):
"""Calculate the cross entropy loss between the predictions and labels.
1 -> 0 will be punished hard, while the other way will not punished not hard.
"""
def calculate_loss(self, predictions, labels, **unused_params):
false_positive_punishment = FLAGS.false_positive_punishment
false_negative_punishment = FLAGS.false_negative_punishment
with tf.name_scope("loss_xent_recall"):
epsilon = 10e-6
if FLAGS.label_smoothing:
float_labels = smoothing(labels)
else:
float_labels = tf.cast(labels, tf.float32)
cross_entropy_loss = false_negative_punishment * float_labels * tf.log(predictions + epsilon) \
+ false_positive_punishment * ( 1 - float_labels) * tf.log(1 - predictions + epsilon)
cross_entropy_loss = tf.negative(cross_entropy_loss)
return tf.reduce_mean(tf.reduce_sum(cross_entropy_loss, 1))
class MeanSquareErrorLoss(BaseLoss):
"""Calculate the MSE loss between the predictions and labels.
"""
def calculate_loss(self, predictions, labels, **unused_params):
with tf.name_scope("loss_xent"):
epsilon = 10e-6
if FLAGS.label_smoothing:
float_labels = smoothing(labels)
else:
float_labels = tf.cast(labels, tf.float32)
mse_loss = tf.square(float_labels - predictions)
return tf.reduce_mean(tf.reduce_sum(mse_loss, 1))
class CrossEntropyLoss(BaseLoss):
"""Calculate the cross entropy loss between the predictions and labels.
"""
def calculate_loss(self, predictions, labels, weights=None, **unused_params):
with tf.name_scope("loss_xent"):
epsilon = 10e-6
if FLAGS.label_smoothing:
float_labels = smoothing(labels)
else:
float_labels = tf.cast(labels, tf.float32)
cross_entropy_loss = float_labels * tf.log(predictions + epsilon) + (
1 - float_labels) * tf.log(1 - predictions + epsilon)
cross_entropy_loss = tf.negative(cross_entropy_loss)
if weights is not None:
print cross_entropy_loss, weights
weighted_loss = tf.einsum("ij,i->ij", cross_entropy_loss, weights)
print "create weighted_loss", weighted_loss
return tf.reduce_mean(tf.reduce_sum(weighted_loss, 1))
else:
return tf.reduce_mean(tf.reduce_sum(cross_entropy_loss, 1))
class HingeLoss(BaseLoss):
"""Calculate the hinge loss between the predictions and labels.
Note the subgradient is used in the backpropagation, and thus the optimization
may converge slower. The predictions trained by the hinge loss are between -1
and +1.
"""
def calculate_loss(self, predictions, labels, b=1.0, **unused_params):
with tf.name_scope("loss_hinge"):
float_labels = tf.cast(labels, tf.float32)
all_zeros = tf.zeros(tf.shape(float_labels), dtype=tf.float32)
all_ones = tf.ones(tf.shape(float_labels), dtype=tf.float32)
sign_labels = tf.subtract(tf.scalar_mul(2, float_labels), all_ones)
hinge_loss = tf.maximum(
all_zeros, tf.scalar_mul(b, all_ones) - sign_labels * predictions)
return tf.reduce_mean(tf.reduce_sum(hinge_loss, 1))
class PairwiseHingeLoss(BaseLoss):
def calculate_loss(self, predictions, labels, margin=0.2, adaptive=3.0, origin=1.0, **unused_params):
batch_size = FLAGS.batch_size
num_classes = FLAGS.num_classes
with tf.name_scope("loss_hinge"):
# get sim_neg
mask = tf.cast(labels, tf.float32)
reverse_mask = 1.0 - mask
min_true_pred = tf.reduce_min((predictions - 1.0) * mask, axis=1, keep_dims=True) + 1.0
mask_wrong = tf.stop_gradient(tf.cast(predictions > (min_true_pred - margin), tf.float32) * reverse_mask)
# get positve samples
int_labels = tf.cast(labels, tf.int32)
sample_labels = tf.unstack(int_labels, num=batch_size, axis=0)
sample_predictions = tf.unstack(predictions, num=batch_size, axis=0)
positive_predictions = []
for sample_label, sample_prediction in zip(sample_labels, sample_predictions):
indices = tf.where(sample_label > 0)
expanded_indices = tf.tile(indices[:,0], [num_classes])[:num_classes]
rand_arrange = tf.random_uniform([num_classes], minval=0, maxval=num_classes, dtype=tf.int32)
positive_indices = tf.stop_gradient(tf.gather(expanded_indices, rand_arrange))
positive_prediction = tf.gather(sample_prediction, positive_indices)
positive_predictions.append(positive_prediction)
positive_predictions = tf.stack(positive_predictions)
# hinge_loss
hinge_loss = tf.maximum(predictions - positive_predictions + margin, 0.0)
adaptive_loss = hinge_loss * mask_wrong
adaptive_loss = tf.reduce_mean(tf.reduce_sum(adaptive_loss, axis=1))
origin_loss = hinge_loss * reverse_mask
origin_loss = tf.reduce_mean(tf.reduce_sum(origin_loss, axis=1))
loss = adaptive * adaptive_loss + origin * origin_loss
return loss
class MixedLoss(BaseLoss):
def calculate_loss(self, predictions, labels, margin=0.2, adaptive=3, **unused_params):
cross_ent_loss = CrossEntropyLoss()
pairwise_loss = PairwiseHingeLoss()
ce_loss = cross_ent_loss.calculate_loss(predictions, labels, **unused_params)
pw_loss = pairwise_loss.calculate_loss(predictions, labels, margin, adaptive=1.0, origin=0.0, **unused_params)
return ce_loss + pw_loss * 0.05
class SoftmaxLoss(BaseLoss):
"""Calculate the softmax loss between the predictions and labels.
The function calculates the loss in the following way: first we feed the
predictions to the softmax activation function and then we calculate
the minus linear dot product between the logged softmax activations and the
normalized ground truth label.
It is an extension to the one-hot label. It allows for more than one positive
labels for each sample.
"""
def calculate_loss(self, predictions, labels, **unused_params):
with tf.name_scope("loss_softmax"):
epsilon = 10e-8
float_labels = tf.cast(labels, tf.float32)
# l1 normalization (labels are no less than 0)
label_rowsum = tf.maximum(
tf.reduce_sum(float_labels, 1, keep_dims=True),
epsilon)
norm_float_labels = tf.div(float_labels, label_rowsum)
softmax_outputs = tf.nn.softmax(predictions)
softmax_loss = tf.negative(tf.reduce_sum(
tf.multiply(norm_float_labels, tf.log(softmax_outputs)), 1))
return tf.reduce_mean(softmax_loss)
class MultiTaskLoss(BaseLoss):
"""This is a vitural loss
"""
def calculate_loss(self, unused_predictions, unused_labels, **unused_params):
raise NotImplementedError()
def get_support(self, labels, support_type=None):
if support_type == None:
support_type = FLAGS.support_type
if "," in support_type:
new_labels = []
for st in support_type.split(","):
new_labels.append(tf.cast(self.get_support(labels, st), dtype=tf.float32))
support_labels = tf.concat(new_labels, axis=1)
return support_labels
elif support_type == "vertical":
num_classes = FLAGS.num_classes
num_verticals = FLAGS.num_verticals
vertical_file = FLAGS.vertical_file
vertical_mapping = np.zeros([num_classes, num_verticals], dtype=np.float32)
float_labels = tf.cast(labels, dtype=tf.float32)
with open(vertical_file) as F:
for line in F:
group = map(int, line.strip().split())
if len(group) == 2:
x, y = group
vertical_mapping[x, y] = 1
vm_init = tf.constant_initializer(vertical_mapping)
vm = tf.get_variable("vm", shape = [num_classes, num_verticals],
trainable=False, initializer=vm_init)
vertical_labels = tf.matmul(float_labels, vm)
return tf.cast(vertical_labels > 0.2, tf.float32)
elif support_type == "frequent":
num_frequents = FLAGS.num_frequents
frequent_labels = tf.slice(labels, begin=[0, 0], size=[-1, num_frequents])
frequent_labels = tf.cast(frequent_labels, dtype=tf.float32)
return frequent_labels
elif support_type == "label":
float_labels = tf.cast(labels, dtype=tf.float32)
return float_labels
else:
raise NotImplementedError()
class MultiTaskCrossEntropyAndSoftmaxLoss(MultiTaskLoss):
"""Calculate the loss between the predictions and labels.
"""
def calculate_loss(self, predictions, support_predictions, labels, **unused_params):
support_labels = self.get_support(labels)
ce_loss_fn = CrossEntropyLoss()
cross_entropy_loss = ce_loss_fn.calculate_loss(predictions, labels, **unused_params)
sm_loss_fn = SoftmaxLoss()
softmax_loss = sm_loss_fn.calculate_loss(support_predictions, support_labels, **unused_params)
return cross_entropy_loss * (1.0 - FLAGS.support_loss_percent) + softmax_loss * FLAGS.support_loss_percent
class MultiTaskCrossEntropyLoss(MultiTaskLoss):
"""Calculate the loss between the predictions and labels.
"""
def calculate_loss(self, predictions, support_predictions, labels, **unused_params):
support_labels = self.get_support(labels)
ce_loss_fn = CrossEntropyLoss()
cross_entropy_loss = ce_loss_fn.calculate_loss(predictions, labels, **unused_params)
cross_entropy_loss2 = ce_loss_fn.calculate_loss(support_predictions, support_labels, **unused_params)
return cross_entropy_loss * (1.0 - FLAGS.support_loss_percent) + cross_entropy_loss2 * FLAGS.support_loss_percent
class BatchAgreementCrossEntropyLoss(BaseLoss):
"""loss that exagerate those points that break the batch-wise order
"""
def calculate_loss(self, predictions, labels, **unused_params):
with tf.name_scope("loss_xent_batch"):
batch_agreement = FLAGS.batch_agreement
epsilon = 10e-6
float_batch_size = float(FLAGS.batch_size)
float_labels = tf.cast(labels, tf.float32)
cross_entropy_loss = float_labels * tf.log(predictions + epsilon) + (
1 - float_labels) * tf.log(1 - predictions + epsilon)
cross_entropy_loss = tf.negative(cross_entropy_loss)
positive_predictions = predictions * float_labels + 1.0 - float_labels
min_pp = tf.reduce_min(positive_predictions)
negative_predictions = predictions * (1.0 - float_labels)
max_np = tf.reduce_max(negative_predictions)
# 1s that fall under 0s
false_negatives = tf.cast(predictions < max_np, tf.float32) * float_labels
num_fn = tf.reduce_sum(false_negatives)
center_fn = tf.reduce_sum(predictions * false_negatives) / num_fn
# 0s that grow over 1s
false_positives = tf.cast(predictions > min_pp, tf.float32) * (1.0 - float_labels)
num_fp = tf.reduce_sum(false_positives)
center_fp = tf.reduce_sum(predictions * false_positives) / num_fp
false_range = tf.maximum(epsilon, max_np - min_pp)
# for 1s that fall under 0s
weight_fn = tf.nn.sigmoid((center_fp - predictions) / false_range * 3.0) * (num_fp / float_batch_size) * false_negatives
# for 0s that grow over 1s
weight_fp = tf.nn.sigmoid((predictions - center_fn) / false_range * 3.0) * (num_fn / float_batch_size) * false_positives
weight = (weight_fn + weight_fp) * batch_agreement + 1.0
print weight
return tf.reduce_mean(tf.reduce_sum(weight * cross_entropy_loss, 1))
class TopKBatchAgreementCrossEntropyLoss(BaseLoss):
"""loss that exagerate those points that break the batch-wise order
"""
def calculate_loss(self, predictions, labels, topk=20, **unused_params):
with tf.name_scope("loss_xent_batch"):
batch_agreement = FLAGS.batch_agreement
epsilon = 10e-6
float_batch_size = float(FLAGS.batch_size)
topk_predictions, _ = tf.nn.top_k(predictions, k=20)
min_topk_predictions = tf.reduce_min(topk_predictions, axis=1, keep_dims=True)
topk_mask = tf.cast(predictions >= min_topk_predictions, dtype=tf.float32)
float_labels = tf.cast(labels, tf.float32)
cross_entropy_loss = float_labels * tf.log(predictions + epsilon) + (
1 - float_labels) * tf.log(1 - predictions + epsilon)
cross_entropy_loss = tf.negative(cross_entropy_loss)
# minimum positive predictions in topk
positive_predictions = (predictions * float_labels * topk_mask) + 1.0 - (float_labels * topk_mask)
min_pp = tf.reduce_min(positive_predictions)
# maximum negative predictions
negative_predictions = predictions * (1.0 - float_labels)
max_np = tf.reduce_max(negative_predictions)
# 1s that fall under top-k
false_negatives = tf.cast(predictions < min_topk_predictions, tf.float32) * float_labels
# 0s that grow over 1s in top-k
false_positives = tf.cast(predictions > min_pp, tf.float32) * (1.0 - float_labels) * topk_mask
weight = (false_negatives + false_positives) * batch_agreement + 1.0
weight = tf.stop_gradient(weight)
print weight
return tf.reduce_mean(tf.reduce_sum(weight * cross_entropy_loss, 1))
class MultiTaskDivergenceCrossEntropyLoss(MultiTaskLoss):
"""Calculate the loss between the predictions and labels.
"""
def calculate_loss(self, predictions, support_predictions, labels, **unused_params):
"""
support_predictions batch_size x num_models x num_classes
predictions = tf.reduce_mean(support_predictions, axis=1)
"""
model_count = tf.shape(support_predictions)[1]
vocab_size = tf.shape(support_predictions)[2]
mean_predictions = tf.reduce_mean(support_predictions, axis=1, keep_dims=True)
support_labels = tf.tile(tf.expand_dims(tf.cast(labels, dtype=tf.float32), axis=1), multiples=[1,model_count,1])
support_means = tf.stop_gradient(tf.tile(mean_predictions, multiples=[1,model_count,1]))
support_predictions = tf.reshape(support_predictions, shape=[-1,model_count*vocab_size])
support_labels = tf.reshape(support_labels, shape=[-1,model_count*vocab_size])
support_means = tf.reshape(support_means, shape=[-1,model_count*vocab_size])
ce_loss_fn = CrossEntropyLoss()
# The cross entropy between predictions and ground truth
cross_entropy_loss = ce_loss_fn.calculate_loss(support_predictions, support_labels, **unused_params)
# The cross entropy between predictions and mean predictions
divergence = ce_loss_fn.calculate_loss(support_predictions, support_means, **unused_params)
loss = cross_entropy_loss * (1.0 - FLAGS.support_loss_percent) - divergence * FLAGS.support_loss_percent
return loss
class MultiTaskDivergenceCrossEntropyAndMSELoss(MultiTaskLoss):
"""Calculate the loss between the predictions and labels.
"""
def calculate_loss(self, predictions, support_predictions, labels, **unused_params):
"""
support_predictions batch_size x num_models x num_classes
predictions = tf.reduce_mean(support_predictions, axis=1)
"""
model_count = tf.shape(support_predictions)[1]
vocab_size = tf.shape(support_predictions)[2]
mean_predictions = tf.reduce_mean(support_predictions, axis=1, keep_dims=True)
support_labels = tf.tile(tf.expand_dims(tf.cast(labels, dtype=tf.float32), axis=1), multiples=[1,model_count,1])
support_means = tf.stop_gradient(tf.tile(mean_predictions, multiples=[1,model_count,1]))
support_predictions = tf.reshape(support_predictions, shape=[-1,model_count*vocab_size])
support_labels = tf.reshape(support_labels, shape=[-1,model_count*vocab_size])
support_means = tf.reshape(support_means, shape=[-1,model_count*vocab_size])
ce_loss_fn = CrossEntropyLoss()
# The cross entropy between predictions and ground truth
cross_entropy_loss = ce_loss_fn.calculate_loss(support_predictions, support_labels, **unused_params)
mse_loss_fn = MeanSquareErrorLoss()
# The square error between predictions and mean predictions
divergence = mse_loss_fn.calculate_loss(support_predictions, support_means, **unused_params)
loss = cross_entropy_loss * (1.0 - FLAGS.support_loss_percent) - divergence * FLAGS.support_loss_percent
return loss
================================================
FILE: youtube-8m-wangheda/mean_average_precision_calculator.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Calculate the mean average precision.
It provides an interface for calculating mean average precision
for an entire list or the top-n ranked items.
Example usages:
We first call the function accumulate many times to process parts of the ranked
list. After processing all the parts, we call peek_map_at_n
to calculate the mean average precision.
```
import random
p = np.array([[random.random() for _ in xrange(50)] for _ in xrange(1000)])
a = np.array([[random.choice([0, 1]) for _ in xrange(50)]
for _ in xrange(1000)])
# mean average precision for 50 classes.
calculator = mean_average_precision_calculator.MeanAveragePrecisionCalculator(
num_class=50)
calculator.accumulate(p, a)
aps = calculator.peek_map_at_n()
```
"""
import numpy
import average_precision_calculator
class MeanAveragePrecisionCalculator(object):
"""This class is to calculate mean average precision.
"""
def __init__(self, num_class):
"""Construct a calculator to calculate the (macro) average precision.
Args:
num_class: A positive Integer specifying the number of classes.
top_n_array: A list of positive integers specifying the top n for each
class. The top n in each class will be used to calculate its average
precision at n.
The size of the array must be num_class.
Raises:
ValueError: An error occurred when num_class is not a positive integer;
or the top_n_array is not a list of positive integers.
"""
if not isinstance(num_class, int) or num_class <= 1:
raise ValueError("num_class must be a positive integer.")
self._ap_calculators = [] # member of AveragePrecisionCalculator
self._num_class = num_class # total number of classes
for i in range(num_class):
self._ap_calculators.append(
average_precision_calculator.AveragePrecisionCalculator())
def accumulate(self, predictions, actuals, num_positives=None):
"""Accumulate the predictions and their ground truth labels.
Args:
predictions: A list of lists storing the prediction scores. The outer
dimension corresponds to classes.
actuals: A list of lists storing the ground truth labels. The dimensions
should correspond to the predictions input. Any value
larger than 0 will be treated as positives, otherwise as negatives.
num_positives: If provided, it is a list of numbers representing the
number of true positives for each class. If not provided, the number of
true positives will be inferred from the 'actuals' array.
Raises:
ValueError: An error occurred when the shape of predictions and actuals
does not match.
"""
if not num_positives:
num_positives = [None for i in predictions.shape[1]]
calculators = self._ap_calculators
for i in range(len(predictions)):
calculators[i].accumulate(predictions[i], actuals[i], num_positives[i])
def clear(self):
for calculator in self._ap_calculators:
calculator.clear()
def is_empty(self):
return ([calculator.heap_size for calculator in self._ap_calculators] ==
[0 for _ in range(self._num_class)])
def peek_map_at_n(self):
"""Peek the non-interpolated mean average precision at n.
Returns:
An array of non-interpolated average precision at n (default 0) for each
class.
"""
aps = [self._ap_calculators[i].peek_ap_at_n()
for i in range(self._num_class)]
return aps
================================================
FILE: youtube-8m-wangheda/model_utils.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains a collection of util functions for model construction.
"""
import numpy
import tensorflow as tf
from tensorflow import logging
from tensorflow import flags
import tensorflow.contrib.slim as slim
def SampleRandomSequence(model_input, num_frames, num_samples):
"""Samples a random sequence of frames of size num_samples.
Args:
model_input: A tensor of size batch_size x max_frames x feature_size
num_frames: A tensor of size batch_size x 1
num_samples: A scalar
Returns:
`model_input`: A tensor of size batch_size x num_samples x feature_size
"""
batch_size = tf.shape(model_input)[0]
frame_index_offset = tf.tile(
tf.expand_dims(tf.range(num_samples), 0), [batch_size, 1])
max_start_frame_index = tf.maximum(num_frames - num_samples, 0)
start_frame_index = tf.cast(
tf.multiply(
tf.random_uniform([batch_size, 1]),
tf.cast(max_start_frame_index + 1, tf.float32)), tf.int32)
frame_index = tf.minimum(start_frame_index + frame_index_offset,
tf.cast(num_frames - 1, tf.int32))
batch_index = tf.tile(
tf.expand_dims(tf.range(batch_size), 1), [1, num_samples])
index = tf.stack([batch_index, frame_index], 2)
return tf.gather_nd(model_input, index)
def SampleRandomFrames(model_input, num_frames, num_samples):
"""Samples a random set of frames of size num_samples.
Args:
model_input: A tensor of size batch_size x max_frames x feature_size
num_frames: A tensor of size batch_size x 1
num_samples: A scalar
Returns:
`model_input`: A tensor of size batch_size x num_samples x feature_size
"""
batch_size = tf.shape(model_input)[0]
frame_index = tf.cast(
tf.multiply(
tf.random_uniform([batch_size, num_samples]),
tf.tile(tf.cast(num_frames, tf.float32), [1, num_samples])), tf.int32)
batch_index = tf.tile(
tf.expand_dims(tf.range(batch_size), 1), [1, num_samples])
index = tf.stack([batch_index, frame_index], 2)
return tf.gather_nd(model_input, index)
def FramePooling(frames, method, **unused_params):
"""Pools over the frames of a video.
Args:
frames: A tensor with shape [batch_size, num_frames, feature_size].
method: "average", "max", "attention", or "none".
Returns:
A tensor with shape [batch_size, feature_size] for average, max, or
attention pooling. A tensor with shape [batch_size*num_frames, feature_size]
for none pooling.
Raises:
ValueError: if method is other than "average", "max", "attention", or
"none".
"""
if method == "average":
return tf.reduce_mean(frames, 1)
elif method == "max":
return tf.reduce_max(frames, 1)
elif method == "none":
feature_size = frames.shape_as_list()[2]
return tf.reshape(frames, [-1, feature_size])
else:
raise ValueError("Unrecognized pooling method: %s" % method)
================================================
FILE: youtube-8m-wangheda/models.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains the base class for models."""
class BaseModel(object):
"""Inherit from this class when implementing new models."""
def create_model(self, unused_model_input, **unused_params):
raise NotImplementedError()
================================================
FILE: youtube-8m-wangheda/readers.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Provides readers configured for different datasets."""
import tensorflow as tf
import utils
from tensorflow import logging
def resize_axis(tensor, axis, new_size, fill_value=0):
"""Truncates or pads a tensor to new_size on on a given axis.
Truncate or extend tensor such that tensor.shape[axis] == new_size. If the
size increases, the padding will be performed at the end, using fill_value.
Args:
tensor: The tensor to be resized.
axis: An integer representing the dimension to be sliced.
new_size: An integer or 0d tensor representing the new value for
tensor.shape[axis].
fill_value: Value to use to fill any new entries in the tensor. Will be
cast to the type of tensor.
Returns:
The resized tensor.
"""
tensor = tf.convert_to_tensor(tensor)
shape = tf.unstack(tf.shape(tensor))
pad_shape = shape[:]
pad_shape[axis] = tf.maximum(0, new_size - shape[axis])
shape[axis] = tf.minimum(shape[axis], new_size)
shape = tf.stack(shape)
resized = tf.concat([
tf.slice(tensor, tf.zeros_like(shape), shape),
tf.fill(tf.stack(pad_shape), tf.cast(fill_value, tensor.dtype))
], axis)
# Update shape.
new_shape = tensor.get_shape().as_list() # A copy is being made.
new_shape[axis] = new_size
resized.set_shape(new_shape)
return resized
class BaseReader(object):
"""Inherit from this class when implementing new readers."""
def prepare_reader(self, unused_filename_queue):
"""Create a thread for generating prediction and label tensors."""
raise NotImplementedError()
class YT8MAggregatedFeatureReader(BaseReader):
"""Reads TFRecords of pre-aggregated Examples.
The TFRecords must contain Examples with a sparse int64 'labels' feature and
a fixed length float32 feature, obtained from the features in 'feature_name'.
The float features are assumed to be an average of dequantized values.
"""
def __init__(self,
num_classes=4716,
feature_sizes=[1024],
feature_names=["mean_inc3"]):
"""Construct a YT8MAggregatedFeatureReader.
Args:
num_classes: a positive integer for the number of classes.
feature_sizes: positive integer(s) for the feature dimensions as a list.
feature_names: the feature name(s) in the tensorflow record as a list.
"""
assert len(feature_names) == len(feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(feature_names), len(feature_sizes))
self.num_classes = num_classes
self.feature_sizes = feature_sizes
self.feature_names = feature_names
def prepare_reader(self, filename_queue, batch_size=1024):
"""Creates a single reader thread for pre-aggregated YouTube 8M Examples.
Args:
filename_queue: A tensorflow queue of filename locations.
Returns:
A tuple of video indexes, features, labels, and padding data.
"""
reader = tf.TFRecordReader()
_, serialized_examples = reader.read_up_to(filename_queue, batch_size)
# set the mapping from the fields to data types in the proto
num_features = len(self.feature_names)
assert num_features > 0, "self.feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(self.feature_names), len(self.feature_sizes))
feature_map = {"video_id": tf.FixedLenFeature([], tf.string),
"labels": tf.VarLenFeature(tf.int64)}
for feature_index in range(num_features):
feature_map[self.feature_names[feature_index]] = tf.FixedLenFeature(
[self.feature_sizes[feature_index]], tf.float32)
features = tf.parse_example(serialized_examples, features=feature_map)
labels = tf.sparse_to_indicator(features["labels"], self.num_classes)
labels.set_shape([None, self.num_classes])
concatenated_features = tf.concat([
features[feature_name] for feature_name in self.feature_names], 1)
return features["video_id"], concatenated_features, labels, tf.ones([tf.shape(serialized_examples)[0]])
class YT8MFrameFeatureReader(BaseReader):
"""Reads TFRecords of SequenceExamples.
The TFRecords must contain SequenceExamples with the sparse in64 'labels'
context feature and a fixed length byte-quantized feature vector, obtained
from the features in 'feature_names'. The quantized features will be mapped
back into a range between min_quantized_value and max_quantized_value.
"""
def __init__(self,
num_classes=4716,
feature_sizes=[1024],
feature_names=["inc3"],
max_frames=300):
"""Construct a YT8MFrameFeatureReader.
Args:
num_classes: a positive integer for the number of classes.
feature_sizes: positive integer(s) for the feature dimensions as a list.
feature_names: the feature name(s) in the tensorflow record as a list.
max_frames: the maximum number of frames to process.
"""
assert len(feature_names) == len(feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(feature_names), len(feature_sizes))
self.num_classes = num_classes
self.feature_sizes = feature_sizes
self.feature_names = feature_names
self.max_frames = max_frames
def get_video_matrix(self,
features,
feature_size,
max_frames,
max_quantized_value,
min_quantized_value):
"""Decodes features from an input string and quantizes it.
Args:
features: raw feature values
feature_size: length of each frame feature vector
max_frames: number of frames (rows) in the output feature_matrix
max_quantized_value: the maximum of the quantized value.
min_quantized_value: the minimum of the quantized value.
Returns:
feature_matrix: matrix of all frame-features
num_frames: number of frames in the sequence
"""
decoded_features = tf.reshape(
tf.cast(tf.decode_raw(features, tf.uint8), tf.float32),
[-1, feature_size])
num_frames = tf.minimum(tf.shape(decoded_features)[0], max_frames)
feature_matrix = utils.Dequantize(decoded_features,
max_quantized_value,
min_quantized_value)
feature_matrix = resize_axis(feature_matrix, 0, max_frames)
return feature_matrix, num_frames
def prepare_reader(self,
filename_queue,
max_quantized_value=2,
min_quantized_value=-2):
"""Creates a single reader thread for YouTube8M SequenceExamples.
Args:
filename_queue: A tensorflow queue of filename locations.
max_quantized_value: the maximum of the quantized value.
min_quantized_value: the minimum of the quantized value.
Returns:
A tuple of video indexes, video features, labels, and padding data.
"""
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
contexts, features = tf.parse_single_sequence_example(
serialized_example,
context_features={"video_id": tf.FixedLenFeature(
[], tf.string),
"labels": tf.VarLenFeature(tf.int64)},
sequence_features={
feature_name : tf.FixedLenSequenceFeature([], dtype=tf.string)
for feature_name in self.feature_names
})
# read ground truth labels
labels = (tf.cast(
tf.sparse_to_dense(contexts["labels"].values, (self.num_classes,), 1,
validate_indices=False),
tf.bool))
# loads (potentially) different types of features and concatenates them
num_features = len(self.feature_names)
assert num_features > 0, "No feature selected: feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(self.feature_names), len(self.feature_sizes))
num_frames = -1 # the number of frames in the video
feature_matrices = [None] * num_features # an array of different features
for feature_index in range(num_features):
feature_matrix, num_frames_in_this_feature = self.get_video_matrix(
features[self.feature_names[feature_index]],
self.feature_sizes[feature_index],
self.max_frames,
max_quantized_value,
min_quantized_value)
if num_frames == -1:
num_frames = num_frames_in_this_feature
else:
tf.assert_equal(num_frames, num_frames_in_this_feature)
feature_matrices[feature_index] = feature_matrix
# cap the number of frames at self.max_frames
num_frames = tf.minimum(num_frames, self.max_frames)
# concatenate different features
video_matrix = tf.concat(feature_matrices, 1)
# convert to batch format.
# TODO: Do proper batch reads to remove the IO bottleneck.
batch_video_ids = tf.expand_dims(contexts["video_id"], 0)
batch_video_matrix = tf.expand_dims(video_matrix, 0)
batch_labels = tf.expand_dims(labels, 0)
batch_frames = tf.expand_dims(num_frames, 0)
return batch_video_ids, batch_video_matrix, batch_labels, batch_frames
class YT8MAggregatedDistillationFeatureReader(BaseReader):
"""Reads TFRecords of pre-aggregated Examples.
The TFRecords must contain Examples with a sparse int64 'labels' feature and
a fixed length float32 feature, obtained from the features in 'feature_name'.
The float features are assumed to be an average of dequantized values.
"""
def __init__(self,
num_classes=4716,
feature_sizes=[1024],
feature_names=["mean_inc3"]):
"""Construct a YT8MAggregatedFeatureReader.
Args:
num_classes: a positive integer for the number of classes.
feature_sizes: positive integer(s) for the feature dimensions as a list.
feature_names: the feature name(s) in the tensorflow record as a list.
"""
assert len(feature_names) == len(feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(feature_names), len(feature_sizes))
self.num_classes = num_classes
self.feature_sizes = feature_sizes
self.feature_names = feature_names
def prepare_reader(self, filename_queue, batch_size=1024):
"""Creates a single reader thread for pre-aggregated YouTube 8M Examples.
Args:
filename_queue: A tensorflow queue of filename locations.
Returns:
A tuple of video indexes, features, labels, and padding data.
"""
reader = tf.TFRecordReader()
_, serialized_examples = reader.read_up_to(filename_queue, batch_size)
# set the mapping from the fields to data types in the proto
num_features = len(self.feature_names)
assert num_features > 0, "self.feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(self.feature_names), len(self.feature_sizes))
feature_map = {"video_id": tf.FixedLenFeature([], tf.string),
"predictions": tf.FixedLenFeature([self.num_classes], tf.float32),
"labels": tf.VarLenFeature(tf.int64)}
for feature_index in range(num_features):
feature_map[self.feature_names[feature_index]] = tf.FixedLenFeature(
[self.feature_sizes[feature_index]], tf.float32)
features = tf.parse_example(serialized_examples, features=feature_map)
labels = tf.sparse_to_indicator(features["labels"], self.num_classes)
labels.set_shape([None, self.num_classes])
concatenated_features = tf.concat([
features[feature_name] for feature_name in self.feature_names], 1)
return features["video_id"], concatenated_features, labels, tf.ones([tf.shape(serialized_examples)[0]]), features["predictions"]
class YT8MFrameDistillationFeatureReader(BaseReader):
"""Reads TFRecords of SequenceExamples.
The TFRecords must contain SequenceExamples with the sparse in64 'labels'
context feature and a fixed length byte-quantized feature vector, obtained
from the features in 'feature_names'. The quantized features will be mapped
back into a range between min_quantized_value and max_quantized_value.
"""
def __init__(self,
num_classes=4716,
feature_sizes=[1024],
feature_names=["inc3"],
max_frames=300):
"""Construct a YT8MFrameFeatureReader.
Args:
num_classes: a positive integer for the number of classes.
feature_sizes: positive integer(s) for the feature dimensions as a list.
feature_names: the feature name(s) in the tensorflow record as a list.
max_frames: the maximum number of frames to process.
"""
assert len(feature_names) == len(feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(feature_names), len(feature_sizes))
self.num_classes = num_classes
self.feature_sizes = feature_sizes
self.feature_names = feature_names
self.max_frames = max_frames
def get_video_matrix(self,
features,
feature_size,
max_frames,
max_quantized_value,
min_quantized_value):
"""Decodes features from an input string and quantizes it.
Args:
features: raw feature values
feature_size: length of each frame feature vector
max_frames: number of frames (rows) in the output feature_matrix
max_quantized_value: the maximum of the quantized value.
min_quantized_value: the minimum of the quantized value.
Returns:
feature_matrix: matrix of all frame-features
num_frames: number of frames in the sequence
"""
decoded_features = tf.reshape(
tf.cast(tf.decode_raw(features, tf.uint8), tf.float32),
[-1, feature_size])
num_frames = tf.minimum(tf.shape(decoded_features)[0], max_frames)
feature_matrix = utils.Dequantize(decoded_features,
max_quantized_value,
min_quantized_value)
feature_matrix = resize_axis(feature_matrix, 0, max_frames)
return feature_matrix, num_frames
def prepare_reader(self,
filename_queue,
max_quantized_value=2,
min_quantized_value=-2):
"""Creates a single reader thread for YouTube8M SequenceExamples.
Args:
filename_queue: A tensorflow queue of filename locations.
max_quantized_value: the maximum of the quantized value.
min_quantized_value: the minimum of the quantized value.
Returns:
A tuple of video indexes, video features, labels, and padding data.
"""
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
contexts, features = tf.parse_single_sequence_example(
serialized_example,
context_features={"video_id": tf.FixedLenFeature([], tf.string),
"predictions": tf.FixedLenFeature([self.num_classes], tf.float32),
"labels": tf.VarLenFeature(tf.int64)},
sequence_features={
feature_name : tf.FixedLenSequenceFeature([], dtype=tf.string)
for feature_name in self.feature_names
})
# read ground truth labels
labels = (tf.cast(
tf.sparse_to_dense(contexts["labels"].values, (self.num_classes,), 1,
validate_indices=False),
tf.bool))
predictions = contexts["predictions"]
# loads (potentially) different types of features and concatenates them
num_features = len(self.feature_names)
assert num_features > 0, "No feature selected: feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(self.feature_names), len(self.feature_sizes))
num_frames = -1 # the number of frames in the video
feature_matrices = [None] * num_features # an array of different features
for feature_index in range(num_features):
feature_matrix, num_frames_in_this_feature = self.get_video_matrix(
features[self.feature_names[feature_index]],
self.feature_sizes[feature_index],
self.max_frames,
max_quantized_value,
min_quantized_value)
if num_frames == -1:
num_frames = num_frames_in_this_feature
else:
tf.assert_equal(num_frames, num_frames_in_this_feature)
feature_matrices[feature_index] = feature_matrix
# cap the number of frames at self.max_frames
num_frames = tf.minimum(num_frames, self.max_frames)
# concatenate different features
video_matrix = tf.concat(feature_matrices, 1)
# convert to batch format.
# TODO: Do proper batch reads to remove the IO bottleneck.
batch_video_ids = tf.expand_dims(contexts["video_id"], 0)
batch_video_matrix = tf.expand_dims(video_matrix, 0)
batch_labels = tf.expand_dims(labels, 0)
batch_frames = tf.expand_dims(num_frames, 0)
batch_predictions = tf.expand_dims(predictions, 0)
return batch_video_ids, batch_video_matrix, batch_labels, batch_frames, batch_predictions
================================================
FILE: youtube-8m-wangheda/train-with-predictions.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for training Tensorflow models on the YouTube-8M dataset."""
import json
import os
import time
import numpy
import eval_util
import losses
import frame_level_models
import video_level_models
import data_augmentation
import feature_transform
import readers
import tensorflow as tf
import tensorflow.contrib.slim as slim
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import utils
FLAGS = flags.FLAGS
if __name__ == "__main__":
# Dataset flags.
flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
"The directory to save the model files in.")
flags.DEFINE_string(
"train_data_pattern", "",
"File glob for the training dataset. If the files refer to Frame Level "
"features (i.e. tensorflow.SequenceExample), then set --reader_type "
"format. The (Sequence)Examples are expected to have 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
flags.DEFINE_string("predictions_data_pattern", None, "File glob for predictions data")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024", "Length of the feature vectors.")
# Distillation flags
flags.DEFINE_bool(
"distillation_features", False,
"If set, *DistillationFeatureReader will be used, the feature must contains"
"prediction features (shape = [4716]).")
flags.DEFINE_integer(
"distillation_type", 0, "Type of distillation, options are 1 and 2.")
flags.DEFINE_bool(
"distillation_as_input", False, "If set true, distillation_predictions will be given to model.")
flags.DEFINE_float("distillation_percent", 0.0,
"If larger than 0, final_loss = distillation_loss * percent + normal_loss * (1.0 - percent).")
# Model flags.
flags.DEFINE_bool(
"frame_features", False,
"If set, then --train_data_pattern must be frame-level features. "
"Otherwise, --train_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_string(
"model", "LogisticModel",
"Which architecture to use for the model. Models are defined "
"in models.py.")
flags.DEFINE_bool(
"multitask", False,
"Whether to consider support_predictions")
flags.DEFINE_bool(
"start_new_model", False,
"If set, this will not resume from a checkpoint and will instead create a"
" new model instance.")
# Training flags.
flags.DEFINE_integer("batch_size", 1024,
"How many examples to process per batch for training.")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Which loss function to use for training the model.")
flags.DEFINE_float(
"regularization_penalty", 1,
"How much weight to give to the regularization loss (the label loss has "
"a weight of 1).")
flags.DEFINE_float("base_learning_rate", 0.01,
"Which learning rate to start with.")
flags.DEFINE_float("learning_rate_decay", 0.95,
"Learning rate decay factor to be applied every "
"learning_rate_decay_examples.")
flags.DEFINE_float("learning_rate_decay_examples", 4000000,
"Multiply current learning rate by learning_rate_decay "
"every learning_rate_decay_examples.")
flags.DEFINE_integer("num_epochs", 5,
"How many passes to make over the dataset before "
"halting training.")
flags.DEFINE_integer("max_steps", None,
"How many steps before stop.")
flags.DEFINE_float("keep_checkpoint_every_n_hours", 1.0,
"How many hours before saving a new checkpoint")
flags.DEFINE_integer("keep_checkpoint_interval", 15,
"How many minutes before saving a new checkpoint")
flags.DEFINE_bool("reweight", False,
"Whether to load model weight from file.")
flags.DEFINE_string("sample_vocab_file", "",
"Where to load video_id vocabulary.")
flags.DEFINE_string("sample_freq_file", "",
"Where to load sample frequency.")
# Other flags.
flags.DEFINE_integer("num_readers", 8,
"How many threads to use for reading input files.")
flags.DEFINE_string("optimizer", "AdamOptimizer",
"What optimizer class to use.")
flags.DEFINE_float("clip_gradient_norm", 1.0, "Norm to clip gradients to.")
flags.DEFINE_bool(
"log_device_placement", False,
"Whether to write the device on which every op will run into the "
"logs on startup.")
flags.DEFINE_integer("recall_at_n", 100,
"N in recall@N.")
flags.DEFINE_bool(
"dropout", False,
"Whether to consider dropout")
flags.DEFINE_float("keep_prob", 1.0,
"probability to keep output (used in dropout, keep it unchanged in validationg and test)")
flags.DEFINE_float("noise_level", 0.0,
"standard deviation of noise (added to hidden nodes)")
def validate_class_name(flag_value, category, modules, expected_superclass):
"""Checks that the given string matches a class of the expected type.
Args:
flag_value: A string naming the class to instantiate.
category: A string used further describe the class in error messages
(e.g. 'model', 'reader', 'loss').
modules: A list of modules to search for the given class.
expected_superclass: A class that the given class should inherit from.
Raises:
FlagsError: If the given class could not be found or if the first class
found with that name doesn't inherit from the expected superclass.
Returns:
True if a class was found that matches the given constraints.
"""
candidates = [getattr(module, flag_value, None) for module in modules]
for candidate in candidates:
if not candidate:
continue
if not issubclass(candidate, expected_superclass):
raise flags.FlagsError("%s '%s' doesn't inherit from %s." %
(category, flag_value,
expected_superclass.__name__))
return True
raise flags.FlagsError("Unable to find %s '%s'." % (category, flag_value))
def get_input_data_tensors(reader,
data_pattern,
batch_size=128,
num_epochs=None):
"""Creates the section of the graph which reads the training data.
Args:
reader: A class which parses the training data.
data_pattern: A 'glob' style path to the data files.
batch_size: How many examples to process at a time.
num_epochs: How many passes to make over the training data. Set to 'None'
to run indefinitely.
num_readers: How many I/O threads to use.
Returns:
A tuple containing the features tensor, labels tensor, and optionally a
tensor containing the number of frames per video. The exact dimensions
depend on the reader being used.
Raises:
IOError: If no files matching the given pattern were found.
"""
logging.info("Using batch size of " + str(batch_size) + " for training.")
with tf.name_scope("train_input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find training files. data_pattern='" +
data_pattern + "'.")
logging.info("Number of training files: %s.", str(len(files)))
files.sort()
filename_queue = tf.train.string_input_producer(
files, num_epochs=num_epochs, shuffle=False)
training_data = reader.prepare_reader(filename_queue)
return tf.train.batch(
training_data,
batch_size=batch_size,
capacity=FLAGS.batch_size * 10,
allow_smaller_final_batch=False,
enqueue_many=True)
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def get_video_weights_array():
weight_lines = open(FLAGS.sample_freq_file).readlines()
weights = numpy.array(map(float, weight_lines))
weights = weights.reshape([len(weight_lines)])
return weights, len(weight_lines)
def optional_assign_weights(sess, weights_input, weights_assignment):
if weights_input is not None:
weights, length = get_video_weights_array()
_ = sess.run(weights_assignment, feed_dict={weights_input: weights})
print "Assigned weights from %s" % FLAGS.sample_freq_file
else:
print "Collection weights_input not found"
def get_video_weights(video_id_batch):
video_id_to_index = tf.contrib.lookup.string_to_index_table_from_file(
vocabulary_file=FLAGS.sample_vocab_file, default_value=0)
indexes = video_id_to_index.lookup(video_id_batch)
weights, length = get_video_weights_array()
weights_input = tf.placeholder(tf.float32, shape=[length], name="sample_weights_input")
weights_tensor = tf.get_variable("sample_weights",
shape=[length],
trainable=False,
dtype=tf.float32,
initializer=tf.constant_initializer(weights))
weights_assignment = tf.assign(weights_tensor, weights_input)
tf.add_to_collection("weights_input", weights_input)
tf.add_to_collection("weights_assignment", weights_assignment)
video_weight_batch = tf.nn.embedding_lookup(weights_tensor, indexes)
return video_weight_batch
def build_graph(reader,
predictions_readers,
train_data_pattern,
predictions_data_patterns,
model,
label_loss_fn=losses.CrossEntropyLoss(),
batch_size=1000,
base_learning_rate=0.01,
learning_rate_decay_examples=1000000,
learning_rate_decay=0.95,
optimizer_class=tf.train.AdamOptimizer,
transformer_class=feature_transform.DefaultTransformer,
augmenter_class=data_augmentation.DefaultAugmenter,
clip_gradient_norm=1.0,
regularization_penalty=1,
num_readers=1,
num_epochs=None):
"""Creates the Tensorflow graph.
This will only be called once in the life of
a training model, because after the graph is created the model will be
restored from a meta graph file rather than being recreated.
Args:
reader: The data file reader. It should inherit from BaseReader.
model: The core model (e.g. logistic or neural net). It should inherit
from BaseModel.
train_data_pattern: glob path to the training data files.
label_loss_fn: What kind of loss to apply to the model. It should inherit
from BaseLoss.
batch_size: How many examples to process at a time.
base_learning_rate: What learning rate to initialize the optimizer with.
optimizer_class: Which optimization algorithm to use.
clip_gradient_norm: Magnitude of the gradient to clip to.
regularization_penalty: How much weight to give the regularization loss
compared to the label loss.
num_readers: How many threads to use for I/O operations.
num_epochs: How many passes to make over the data. 'None' means an
unlimited number of passes.
"""
global_step = tf.Variable(0, trainable=False, name="global_step")
learning_rate = tf.train.exponential_decay(
base_learning_rate,
global_step * batch_size,
learning_rate_decay_examples,
learning_rate_decay,
staircase=True)
tf.summary.scalar('learning_rate', learning_rate)
optimizer = optimizer_class(learning_rate)
all_distill_labels = []
for dreader, dpattern in zip(predictions_readers, predictions_data_patterns):
distill_video_id, distill_labels_batch, unused_labels_batch, unused_num_frames = (
get_input_data_tensors(
dreader,
dpattern,
batch_size=batch_size,
num_epochs=num_epochs))
all_distill_labels.append(distill_labels_batch)
all_distill_labels = tf.stack(all_distill_labels, axis=2)
distill_weight_var = tf.get_variable("distill_weight", [len(predictions_readers)])
distill_weight = tf.nn.softmax(distill_weight_var)
final_distill_labels = tf.einsum("ijk,k->ij", all_distill_labels, distill_weight)
if FLAGS.distillation_features:
video_id, model_input_raw, labels_batch, num_frames, distill_labels_batch = (
get_input_data_tensors(
reader,
train_data_pattern,
batch_size=batch_size,
num_epochs=num_epochs))
if FLAGS.distillation_features and FLAGS.distillation_type == 2:
p = FLAGS.distillation_percent
print "distillation_percent =", p, "reforming labels"
float_labels = tf.cast(labels_batch, dtype=tf.float32)
sum_float_labels = tf.reduce_sum(float_labels, axis=1, keep_dims=True)
sum_distill_labels = tf.reduce_sum(distill_labels_batch, axis=1, keep_dims=True) + 1e-6
distill_labels_batch = float_labels + distill_labels_batch * (sum_float_labels / sum_distill_labels * p)
distill_labels_batch = tf.clip_by_value(distill_labels_batch, clip_value_min=0.0, clip_value_max=1.0)
else:
video_id, model_input_raw, labels_batch, num_frames = (
get_input_data_tensors(
reader,
train_data_pattern,
batch_size=batch_size,
num_epochs=num_epochs))
id_mismatch = tf.reduce_mean(tf.cast(tf.not_equal(video_id, distill_video_id), tf.float32))
# data augmentation, will not persist in inference
data_augmenter = augmenter_class()
model_input_raw, labels_batch, num_frames = data_augmenter.augment(model_input_raw, num_frames=num_frames, labels_batch=labels_batch)
tf.summary.histogram("model/input_raw", model_input_raw)
feature_transformer = transformer_class()
model_input, num_frames = feature_transformer.transform(model_input_raw, num_frames=num_frames)
tf.summary.histogram("model/input", model_input)
tf.summary.scalar("model/id_mismatch", id_mismatch)
with tf.name_scope("model"):
if FLAGS.noise_level > 0:
noise_level_tensor = tf.placeholder_with_default(0.0, shape=[], name="noise_level")
else:
noise_level_tensor = None
if FLAGS.distillation_as_input:
distillation_predictions = final_distill_labels
else:
distillation_predictions = None
if FLAGS.dropout:
keep_prob_tensor = tf.placeholder_with_default(1.0, shape=[], name="keep_prob")
result = model.create_model(
model_input,
num_frames=num_frames,
vocab_size=reader.num_classes,
labels=labels_batch,
dropout=FLAGS.dropout,
keep_prob=keep_prob_tensor,
distillation_predictions=distillation_predictions,
noise_level=noise_level_tensor)
else:
result = model.create_model(
model_input,
num_frames=num_frames,
vocab_size=reader.num_classes,
labels=labels_batch,
distillation_predictions=distillation_predictions,
noise_level=noise_level_tensor)
for variable in slim.get_model_variables():
tf.summary.histogram(variable.op.name, variable)
print "result", result
predictions = result["predictions"]
if "loss" in result.keys():
label_loss = result["loss"]
else:
video_weights_batch = None
if FLAGS.reweight:
video_weights_batch = get_video_weights(video_id)
if FLAGS.multitask:
support_predictions = result["support_predictions"]
tf.summary.histogram("model/support_predictions", support_predictions)
print "support_predictions", support_predictions
if FLAGS.distillation_features and FLAGS.distillation_type == 1:
p = FLAGS.distillation_percent
print "distillation_percent =", p
if p <= 0:
label_loss = label_loss_fn.calculate_loss(predictions, support_predictions, labels_batch, weights=video_weights_batch)
elif p >= 1:
label_loss = label_loss_fn.calculate_loss(predictions, support_predictions, distill_labels_batch, weights=video_weights_batch)
else:
label_loss = label_loss_fn.calculate_loss(predictions, support_predictions, labels_batch, weights=video_weights_batch) * (1.0 - p) \
+ label_loss_fn.calculate_loss(predictions, support_predictions, distill_labels_batch, weights=video_weights_batch) * p
elif FLAGS.distillation_features and FLAGS.distillation_type == 2:
print "using pure distillation loss"
label_loss = label_loss_fn.calculate_loss(predictions, support_predictions, distill_labels_batch, weights=video_weights_batch)
else:
print "using original loss"
label_loss = label_loss_fn.calculate_loss(predictions, support_predictions, labels_batch, weights=video_weights_batch)
else:
if FLAGS.distillation_features and FLAGS.distillation_type == 1:
p = FLAGS.distillation_percent
print "distillation_percent =", p
if p <= 0:
label_loss = label_loss_fn.calculate_loss(predictions, labels_batch, weights=video_weights_batch)
elif p >= 1:
label_loss = label_loss_fn.calculate_loss(predictions, distill_labels_batch, weights=video_weights_batch)
else:
label_loss = label_loss_fn.calculate_loss(predictions, labels_batch, weights=video_weights_batch) * (1.0 - p) \
+ label_loss_fn.calculate_loss(predictions, distill_labels_batch, weights=video_weights_batch) * p
elif FLAGS.distillation_features and FLAGS.distillation_type == 2:
print "using pure distillation loss"
label_loss = label_loss_fn.calculate_loss(predictions, distill_labels_batch, weights=video_weights_batch)
else:
print "using original loss"
label_loss = label_loss_fn.calculate_loss(predictions, labels_batch, weights=video_weights_batch)
tf.summary.histogram("model/predictions", predictions)
tf.summary.scalar("label_loss", label_loss)
if "regularization_loss" in result.keys():
reg_loss = result["regularization_loss"]
else:
reg_loss = tf.constant(0.0)
reg_losses = tf.losses.get_regularization_losses()
if reg_losses:
reg_loss += tf.add_n(reg_losses)
if regularization_penalty != 0:
tf.summary.scalar("reg_loss", reg_loss)
# Adds update_ops (e.g., moving average updates in batch normalization) as
# a dependency to the train_op.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
if "update_ops" in result.keys():
update_ops += result["update_ops"]
if update_ops:
with tf.control_dependencies(update_ops):
barrier = tf.no_op(name="gradient_barrier")
with tf.control_dependencies([barrier]):
label_loss = tf.identity(label_loss)
# Incorporate the L2 weight penalties etc.
final_loss = regularization_penalty * reg_loss + label_loss
gradients = optimizer.compute_gradients(final_loss,
colocate_gradients_with_ops=False)
if clip_gradient_norm > 0:
with tf.name_scope('clip_grads'):
gradients = utils.clip_gradient_norms(gradients , clip_gradient_norm)
train_op = optimizer.apply_gradients(gradients, global_step=global_step)
tf.add_to_collection("global_step", global_step)
tf.add_to_collection("loss", label_loss)
tf.add_to_collection("predictions", predictions)
tf.add_to_collection("input_batch_raw", model_input_raw)
tf.add_to_collection("input_batch", model_input)
tf.add_to_collection("num_frames", num_frames)
tf.add_to_collection("labels", tf.cast(labels_batch, tf.float32))
tf.add_to_collection("train_op", train_op)
if FLAGS.dropout:
tf.add_to_collection("keep_prob", keep_prob_tensor)
if FLAGS.noise_level > 0:
tf.add_to_collection("noise_level", noise_level_tensor)
class Trainer(object):
"""A Trainer to train a Tensorflow graph."""
def __init__(self, cluster, task, train_dir, log_device_placement=True):
""""Creates a Trainer.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
self.cluster = cluster
self.task = task
self.is_master = (task.type == "master" and task.index == 0)
self.train_dir = train_dir
self.config = tf.ConfigProto(log_device_placement=log_device_placement)
if self.is_master and self.task.index > 0:
raise StandardError("%s: Only one replica of master expected",
task_as_string(self.task))
def run(self, start_new_model=False):
"""Performs training on the currently defined Tensorflow graph.
Returns:
A tuple of the training Hit@1 and the training PERR.
"""
if self.is_master and start_new_model:
self.remove_training_directory(self.train_dir)
target, device_fn = self.start_server_if_distributed()
meta_filename = self.get_meta_filename(start_new_model, self.train_dir)
with tf.Graph().as_default() as graph:
if meta_filename:
saver = self.recover_model(meta_filename)
with tf.device(device_fn):
if not meta_filename:
saver = self.build_model()
global_step = tf.get_collection("global_step")[0]
loss = tf.get_collection("loss")[0]
predictions = tf.get_collection("predictions")[0]
labels = tf.get_collection("labels")[0]
train_op = tf.get_collection("train_op")[0]
init_op = tf.global_variables_initializer()
if FLAGS.dropout:
keep_prob_tensor = tf.get_collection("keep_prob")[0]
if FLAGS.noise_level > 0:
noise_level_tensor = tf.get_collection("noise_level")[0]
if FLAGS.reweight:
weights_input, weights_assignment = None, None
if len(tf.get_collection("weights_input")) > 0:
weights_input = tf.get_collection("weights_input")[0]
weights_assignment = tf.get_collection("weights_assignment")[0]
sv = tf.train.Supervisor(
graph,
logdir=self.train_dir,
init_op=init_op,
is_chief=self.is_master,
global_step=global_step,
save_model_secs=FLAGS.keep_checkpoint_interval * 60,
save_summaries_secs=120,
saver=saver)
logging.info("%s: Starting managed session.", task_as_string(self.task))
with sv.managed_session(target, config=self.config) as sess:
# re-assign weights
if FLAGS.reweight:
optional_assign_weights(sess, weights_input, weights_assignment)
steps = 0
try:
logging.info("%s: Entering training loop.", task_as_string(self.task))
while not sv.should_stop():
steps += 1
batch_start_time = time.time()
custom_feed = {}
if FLAGS.dropout:
custom_feed[keep_prob_tensor] = FLAGS.keep_prob
if FLAGS.noise_level > 0:
custom_feed[noise_level_tensor] = FLAGS.noise_level
_, global_step_val, loss_val, predictions_val, labels_val = sess.run(
[train_op, global_step, loss, predictions, labels], feed_dict=custom_feed)
seconds_per_batch = time.time() - batch_start_time
if self.is_master:
examples_per_second = labels_val.shape[0] / seconds_per_batch
hit_at_one = eval_util.calculate_hit_at_one(predictions_val,
labels_val)
perr = eval_util.calculate_precision_at_equal_recall_rate(
predictions_val, labels_val)
recall = "N/A"
if False:
recall = eval_util.calculate_recall_at_n(
predictions_val, labels_val, FLAGS.recall_at_n)
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_Recall@%d" % FLAGS.recall_at_n, recall), global_step_val)
recall = "%.2f" % recall
gap = eval_util.calculate_gap(predictions_val, labels_val)
logging.info(
"%s: training step " + str(global_step_val) + "| Hit@1: " +
("%.2f" % hit_at_one) + " PERR: " + ("%.2f" % perr) + " GAP: " +
("%.2f" % gap) + " Recall@%d: " % FLAGS.recall_at_n +
recall + " Loss: " + str(loss_val),
task_as_string(self.task))
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_Hit@1", hit_at_one),
global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_Perr", perr), global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_GAP", gap), global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("global_step/Examples/Second",
examples_per_second), global_step_val)
sv.summary_writer.flush()
if FLAGS.max_steps is not None and steps > FLAGS.max_steps:
logging.info("%s: Done training -- max_steps limit reached.",
task_as_string(self.task))
break
except tf.errors.OutOfRangeError:
logging.info("%s: Done training -- epoch limit reached.",
task_as_string(self.task))
logging.info("%s: Exited training loop.", task_as_string(self.task))
sv.Stop()
def start_server_if_distributed(self):
"""Starts a server if the execution is distributed."""
if self.cluster:
logging.info("%s: Starting trainer within cluster %s.",
task_as_string(self.task), self.cluster.as_dict())
server = start_server(self.cluster, self.task)
target = server.target
device_fn = tf.train.replica_device_setter(
ps_device="/job:ps",
worker_device="/job:%s/task:%d" % (self.task.type, self.task.index),
cluster=self.cluster)
else:
target = ""
device_fn = ""
return (target, device_fn)
def remove_training_directory(self, train_dir):
"""Removes the training directory."""
try:
logging.info(
"%s: Removing existing train directory.",
task_as_string(self.task))
gfile.DeleteRecursively(train_dir)
except:
logging.error(
"%s: Failed to delete directory " + train_dir +
" when starting a new model. Please delete it manually and" +
" try again.", task_as_string(self.task))
def get_meta_filename(self, start_new_model, train_dir):
if start_new_model:
logging.info("%s: Flag 'start_new_model' is set. Building a new model.",
task_as_string(self.task))
return None
latest_checkpoint = tf.train.latest_checkpoint(train_dir)
if not latest_checkpoint:
logging.info("%s: No checkpoint file found. Building a new model.",
task_as_string(self.task))
return None
meta_filename = latest_checkpoint + ".meta"
if not gfile.Exists(meta_filename):
logging.info("%s: No meta graph file found. Building a new model.",
task_as_string(self.task))
return None
else:
return meta_filename
def recover_model(self, meta_filename):
logging.info("%s: Restoring from meta graph file %s",
task_as_string(self.task), meta_filename)
return tf.train.import_meta_graph(meta_filename)
def build_model(self):
"""Find the model and build the graph."""
# Convert feature_names and feature_sizes to lists of values.
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.distillation_features:
print "distillation readers"
if FLAGS.frame_features:
reader = readers.YT8MFrameDistillationFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedDistillationFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
assert FLAGS.predictions_data_pattern is not None, "predictions data must be provided"
predictions_patterns = FLAGS.predictions_data_pattern.strip().split(",")
predictions_readers = []
for pattern in predictions_patterns:
predictions_readers.append(readers.YT8MAggregatedFeatureReader(
feature_names=["predictions"], feature_sizes=[4716]))
# Find the model.
model = find_class_by_name(FLAGS.model,
[frame_level_models, video_level_models])()
label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])()
optimizer_class = find_class_by_name(FLAGS.optimizer, [tf.train])
transformer_class = find_class_by_name(FLAGS.feature_transformer, [feature_transform])
augmenter_class = find_class_by_name(FLAGS.data_augmenter, [data_augmentation])
build_graph(reader=reader,
predictions_readers=predictions_readers,
train_data_pattern=FLAGS.train_data_pattern,
predictions_data_patterns=predictions_patterns,
model=model,
optimizer_class=optimizer_class,
augmenter_class=augmenter_class,
transformer_class=transformer_class,
clip_gradient_norm=FLAGS.clip_gradient_norm,
label_loss_fn=label_loss_fn,
base_learning_rate=FLAGS.base_learning_rate,
learning_rate_decay=FLAGS.learning_rate_decay,
learning_rate_decay_examples=FLAGS.learning_rate_decay_examples,
regularization_penalty=FLAGS.regularization_penalty,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size,
num_epochs=FLAGS.num_epochs)
logging.info("%s: Built graph.", task_as_string(self.task))
return tf.train.Saver(max_to_keep=3, keep_checkpoint_every_n_hours=FLAGS.keep_checkpoint_every_n_hours)
class ParameterServer(object):
"""A parameter server to serve variables in a distributed execution."""
def __init__(self, cluster, task):
"""Creates a ParameterServer.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
self.cluster = cluster
self.task = task
def run(self):
"""Starts the parameter server."""
logging.info("%s: Starting parameter server within cluster %s.",
task_as_string(self.task), self.cluster.as_dict())
server = start_server(self.cluster, self.task)
server.join()
def start_server(cluster, task):
"""Creates a Server.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
if not task.type:
raise ValueError("%s: The task type must be specified." %
task_as_string(task))
if task.index is None:
raise ValueError("%s: The task index must be specified." %
task_as_string(task))
# Create and start a server.
return tf.train.Server(
tf.train.ClusterSpec(cluster),
protocol="grpc",
job_name=task.type,
task_index=task.index)
def task_as_string(task):
return "/job:%s/task:%s" % (task.type, task.index)
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object,), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.",
task_as_string(task), tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
Trainer(cluster, task, FLAGS.train_dir, FLAGS.log_device_placement).run(
start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-wangheda/train-with-rebuild.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for training Tensorflow models on the YouTube-8M dataset."""
import json
import os
import time
import numpy
import eval_util
import losses
import frame_level_models
import video_level_models
import data_augmentation
import feature_transform
import readers
import tensorflow as tf
import tensorflow.contrib.slim as slim
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import utils
FLAGS = flags.FLAGS
if __name__ == "__main__":
# Dataset flags.
flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
"The directory to save the model files in.")
flags.DEFINE_string(
"train_data_pattern", "",
"File glob for the training dataset. If the files refer to Frame Level "
"features (i.e. tensorflow.SequenceExample), then set --reader_type "
"format. The (Sequence)Examples are expected to have 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024", "Length of the feature vectors.")
# Distillation flags
flags.DEFINE_bool(
"distillation_features", False,
"If set, *DistillationFeatureReader will be used, the feature must contains"
"prediction features (shape = [4716]).")
flags.DEFINE_integer(
"distillation_type", 0, "Type of distillation, options are 1 and 2.")
flags.DEFINE_float("distillation_percent", 0.0,
"If larger than 0, final_loss = distillation_loss * percent + normal_loss * (1.0 - percent).")
# Model flags.
flags.DEFINE_bool(
"frame_features", False,
"If set, then --train_data_pattern must be frame-level features. "
"Otherwise, --train_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_string(
"model", "LogisticModel",
"Which architecture to use for the model. Models are defined "
"in models.py.")
flags.DEFINE_bool(
"multitask", False,
"Whether to consider support_predictions")
flags.DEFINE_bool(
"start_new_model", False,
"If set, this will not resume from a checkpoint and will instead create a"
" new model instance.")
# Training flags.
flags.DEFINE_integer("batch_size", 1024,
"How many examples to process per batch for training.")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Which loss function to use for training the model.")
flags.DEFINE_float(
"regularization_penalty", 1,
"How much weight to give to the regularization loss (the label loss has "
"a weight of 1).")
flags.DEFINE_float("base_learning_rate", 0.01,
"Which learning rate to start with.")
flags.DEFINE_float("learning_rate_decay", 0.95,
"Learning rate decay factor to be applied every "
"learning_rate_decay_examples.")
flags.DEFINE_float("learning_rate_decay_examples", 4000000,
"Multiply current learning rate by learning_rate_decay "
"every learning_rate_decay_examples.")
flags.DEFINE_integer("num_epochs", 5,
"How many passes to make over the dataset before "
"halting training.")
flags.DEFINE_integer("max_steps", None,
"How many steps before stop.")
flags.DEFINE_float("keep_checkpoint_every_n_hours", 1.0,
"How many hours before saving a new checkpoint")
flags.DEFINE_bool("reweight", False,
"Whether to load model weight from file.")
flags.DEFINE_string("sample_vocab_file", "",
"Where to load video_id vocabulary.")
flags.DEFINE_string("sample_freq_file", "",
"Where to load sample frequency.")
# Other flags.
flags.DEFINE_integer("num_readers", 8,
"How many threads to use for reading input files.")
flags.DEFINE_string("optimizer", "AdamOptimizer",
"What optimizer class to use.")
flags.DEFINE_float("clip_gradient_norm", 1.0, "Norm to clip gradients to.")
flags.DEFINE_bool(
"log_device_placement", False,
"Whether to write the device on which every op will run into the "
"logs on startup.")
flags.DEFINE_integer("recall_at_n", 100,
"N in recall@N.")
flags.DEFINE_bool(
"dropout", False,
"Whether to consider dropout")
flags.DEFINE_float("keep_prob", 1.0,
"probability to keep output (used in dropout, keep it unchanged in validationg and test)")
flags.DEFINE_float("noise_level", 0.0,
"standard deviation of noise (added to hidden nodes)")
def validate_class_name(flag_value, category, modules, expected_superclass):
"""Checks that the given string matches a class of the expected type.
Args:
flag_value: A string naming the class to instantiate.
category: A string used further describe the class in error messages
(e.g. 'model', 'reader', 'loss').
modules: A list of modules to search for the given class.
expected_superclass: A class that the given class should inherit from.
Raises:
FlagsError: If the given class could not be found or if the first class
found with that name doesn't inherit from the expected superclass.
Returns:
True if a class was found that matches the given constraints.
"""
candidates = [getattr(module, flag_value, None) for module in modules]
for candidate in candidates:
if not candidate:
continue
if not issubclass(candidate, expected_superclass):
raise flags.FlagsError("%s '%s' doesn't inherit from %s." %
(category, flag_value,
expected_superclass.__name__))
return True
raise flags.FlagsError("Unable to find %s '%s'." % (category, flag_value))
def get_input_data_tensors(reader,
data_pattern,
batch_size=1000,
num_epochs=None,
num_readers=1):
"""Creates the section of the graph which reads the training data.
Args:
reader: A class which parses the training data.
data_pattern: A 'glob' style path to the data files.
batch_size: How many examples to process at a time.
num_epochs: How many passes to make over the training data. Set to 'None'
to run indefinitely.
num_readers: How many I/O threads to use.
Returns:
A tuple containing the features tensor, labels tensor, and optionally a
tensor containing the number of frames per video. The exact dimensions
depend on the reader being used.
Raises:
IOError: If no files matching the given pattern were found.
"""
logging.info("Using batch size of " + str(batch_size) + " for training.")
with tf.name_scope("train_input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find training files. data_pattern='" +
data_pattern + "'.")
logging.info("Number of training files: %s.", str(len(files)))
filename_queue = tf.train.string_input_producer(
files, num_epochs=num_epochs, shuffle=True)
training_data = [
reader.prepare_reader(filename_queue) for _ in range(num_readers)
]
return tf.train.shuffle_batch_join(
training_data,
batch_size=batch_size,
capacity=FLAGS.batch_size * 8,
min_after_dequeue=FLAGS.batch_size,
allow_smaller_final_batch=True,
enqueue_many=True)
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def get_video_weights_array():
weight_lines = open(FLAGS.sample_freq_file).readlines()
weights = numpy.array(map(float, weight_lines))
weights = weights.reshape([len(weight_lines)])
return weights, len(weight_lines)
def optional_assign_weights(sess, weights_input, weights_assignment):
if weights_input is not None:
weights, length = get_video_weights_array()
_ = sess.run(weights_assignment, feed_dict={weights_input: weights})
print "Assigned weights from %s" % FLAGS.sample_freq_file
else:
print "Collection weights_input not found"
def get_video_weights(video_id_batch):
video_id_to_index = tf.contrib.lookup.string_to_index_table_from_file(
vocabulary_file=FLAGS.sample_vocab_file, default_value=0)
indexes = video_id_to_index.lookup(video_id_batch)
weights, length = get_video_weights_array()
weights_input = tf.placeholder(tf.float32, shape=[length], name="sample_weights_input")
weights_tensor = tf.get_variable("sample_weights",
shape=[length],
trainable=False,
dtype=tf.float32,
initializer=tf.constant_initializer(weights))
weights_assignment = tf.assign(weights_tensor, weights_input)
tf.add_to_collection("weights_input", weights_input)
tf.add_to_collection("weights_assignment", weights_assignment)
video_weight_batch = tf.nn.embedding_lookup(weights_tensor, indexes)
return video_weight_batch
def build_graph(reader,
model,
train_data_pattern,
label_loss_fn=losses.CrossEntropyLoss(),
batch_size=1000,
base_learning_rate=0.01,
learning_rate_decay_examples=1000000,
learning_rate_decay=0.95,
optimizer_class=tf.train.AdamOptimizer,
transformer_class=feature_transform.DefaultTransformer,
augmenter_class=data_augmentation.DefaultAugmenter,
clip_gradient_norm=1.0,
regularization_penalty=1,
num_readers=1,
num_epochs=None):
"""Creates the Tensorflow graph.
This will only be called once in the life of
a training model, because after the graph is created the model will be
restored from a meta graph file rather than being recreated.
Args:
reader: The data file reader. It should inherit from BaseReader.
model: The core model (e.g. logistic or neural net). It should inherit
from BaseModel.
train_data_pattern: glob path to the training data files.
label_loss_fn: What kind of loss to apply to the model. It should inherit
from BaseLoss.
batch_size: How many examples to process at a time.
base_learning_rate: What learning rate to initialize the optimizer with.
optimizer_class: Which optimization algorithm to use.
clip_gradient_norm: Magnitude of the gradient to clip to.
regularization_penalty: How much weight to give the regularization loss
compared to the label loss.
num_readers: How many threads to use for I/O operations.
num_epochs: How many passes to make over the data. 'None' means an
unlimited number of passes.
"""
global_step = tf.Variable(0, trainable=False, name="global_step")
learning_rate = tf.train.exponential_decay(
base_learning_rate,
global_step * batch_size,
learning_rate_decay_examples,
learning_rate_decay,
staircase=True)
tf.summary.scalar('learning_rate', learning_rate)
optimizer = optimizer_class(learning_rate)
if FLAGS.distillation_features:
video_id, model_input_raw, labels_batch, num_frames, distill_labels_batch = (
get_input_data_tensors(
reader,
train_data_pattern,
batch_size=batch_size,
num_readers=num_readers,
num_epochs=num_epochs))
if FLAGS.distillation_features and FLAGS.distillation_type == 2:
p = FLAGS.distillation_percent
print "distillation_percent =", p, "reforming labels"
float_labels = tf.cast(labels_batch, dtype=tf.float32)
sum_float_labels = tf.reduce_sum(float_labels, axis=1, keep_dims=True)
sum_distill_labels = tf.reduce_sum(distill_labels_batch, axis=1, keep_dims=True) + 1e-6
distill_labels_batch = float_labels + distill_labels_batch * (sum_float_labels / sum_distill_labels * p)
distill_labels_batch = tf.clip_by_value(distill_labels_batch, clip_value_min=0.0, clip_value_max=1.0)
else:
video_id, model_input_raw, labels_batch, num_frames = (
get_input_data_tensors(
reader,
train_data_pattern,
batch_size=batch_size,
num_readers=num_readers,
num_epochs=num_epochs))
# data augmentation, will not persist in inference
data_augmenter = augmenter_class()
model_input_raw, labels_batch, num_frames = data_augmenter.augment(model_input_raw, num_frames=num_frames, labels_batch=labels_batch)
tf.summary.histogram("model/input_raw", model_input_raw)
feature_transformer = transformer_class()
model_input, num_frames = feature_transformer.transform(model_input_raw, num_frames=num_frames)
tf.summary.histogram("model/input", model_input)
with tf.name_scope("model"):
if FLAGS.noise_level > 0:
noise_level_tensor = tf.placeholder_with_default(0.0, shape=[], name="noise_level")
else:
noise_level_tensor = None
if FLAGS.dropout:
keep_prob_tensor = tf.placeholder_with_default(1.0, shape=[], name="keep_prob")
result = model.create_model(
model_input,
num_frames=num_frames,
vocab_size=reader.num_classes,
labels=labels_batch,
dropout=FLAGS.dropout,
keep_prob=keep_prob_tensor,
noise_level=noise_level_tensor)
else:
result = model.create_model(
model_input,
num_frames=num_frames,
vocab_size=reader.num_classes,
labels=labels_batch,
noise_level=noise_level_tensor)
for variable in slim.get_model_variables():
tf.summary.histogram(variable.op.name, variable)
print "result", result
predictions = result["predictions"]
if "loss" in result.keys():
label_loss = result["loss"]
else:
video_weights_batch = None
if FLAGS.reweight:
video_weights_batch = get_video_weights(video_id)
if FLAGS.multitask:
support_predictions = result["support_predictions"]
tf.summary.histogram("model/support_predictions", support_predictions)
print "support_predictions", support_predictions
if FLAGS.distillation_features and FLAGS.distillation_type == 1:
p = FLAGS.distillation_percent
print "distillation_percent =", p
if p <= 0:
label_loss = label_loss_fn.calculate_loss(predictions, support_predictions, labels_batch, weights=video_weights_batch)
elif p >= 1:
label_loss = label_loss_fn.calculate_loss(predictions, support_predictions, distill_labels_batch, weights=video_weights_batch)
else:
label_loss = label_loss_fn.calculate_loss(predictions, support_predictions, labels_batch, weights=video_weights_batch) * (1.0 - p) \
+ label_loss_fn.calculate_loss(predictions, support_predictions, distill_labels_batch, weights=video_weights_batch) * p
elif FLAGS.distillation_features and FLAGS.distillation_type == 2:
print "using pure distillation loss"
label_loss = label_loss_fn.calculate_loss(predictions, support_predictions, distill_labels_batch, weights=video_weights_batch)
else:
print "using original loss"
label_loss = label_loss_fn.calculate_loss(predictions, support_predictions, labels_batch, weights=video_weights_batch)
else:
if FLAGS.distillation_features and FLAGS.distillation_type == 1:
p = FLAGS.distillation_percent
print "distillation_percent =", p
if p <= 0:
label_loss = label_loss_fn.calculate_loss(predictions, labels_batch, weights=video_weights_batch)
elif p >= 1:
label_loss = label_loss_fn.calculate_loss(predictions, distill_labels_batch, weights=video_weights_batch)
else:
label_loss = label_loss_fn.calculate_loss(predictions, labels_batch, weights=video_weights_batch) * (1.0 - p) \
+ label_loss_fn.calculate_loss(predictions, distill_labels_batch, weights=video_weights_batch) * p
elif FLAGS.distillation_features and FLAGS.distillation_type == 2:
label_loss = label_loss_fn.calculate_loss(predictions, distill_labels_batch, weights=video_weights_batch)
else:
label_loss = label_loss_fn.calculate_loss(predictions, labels_batch, weights=video_weights_batch)
tf.summary.histogram("model/predictions", predictions)
tf.summary.scalar("label_loss", label_loss)
if "regularization_loss" in result.keys():
reg_loss = result["regularization_loss"]
else:
reg_loss = tf.constant(0.0)
reg_losses = tf.losses.get_regularization_losses()
if reg_losses:
reg_loss += tf.add_n(reg_losses)
if regularization_penalty != 0:
tf.summary.scalar("reg_loss", reg_loss)
# Adds update_ops (e.g., moving average updates in batch normalization) as
# a dependency to the train_op.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
if "update_ops" in result.keys():
update_ops += result["update_ops"]
if update_ops:
with tf.control_dependencies(update_ops):
barrier = tf.no_op(name="gradient_barrier")
with tf.control_dependencies([barrier]):
label_loss = tf.identity(label_loss)
# Incorporate the L2 weight penalties etc.
final_loss = regularization_penalty * reg_loss + label_loss
gradients = optimizer.compute_gradients(final_loss,
colocate_gradients_with_ops=False)
if clip_gradient_norm > 0:
with tf.name_scope('clip_grads'):
gradients = utils.clip_gradient_norms(gradients , clip_gradient_norm)
train_op = optimizer.apply_gradients(gradients, global_step=global_step)
tf.add_to_collection("global_step", global_step)
tf.add_to_collection("loss", label_loss)
tf.add_to_collection("predictions", predictions)
tf.add_to_collection("input_batch_raw", model_input_raw)
tf.add_to_collection("input_batch", model_input)
tf.add_to_collection("num_frames", num_frames)
tf.add_to_collection("labels", tf.cast(labels_batch, tf.float32))
tf.add_to_collection("train_op", train_op)
if FLAGS.dropout:
tf.add_to_collection("keep_prob", keep_prob_tensor)
if FLAGS.noise_level > 0:
tf.add_to_collection("noise_level", noise_level_tensor)
class Trainer(object):
"""A Trainer to train a Tensorflow graph."""
def __init__(self, cluster, task, train_dir, log_device_placement=True):
""""Creates a Trainer.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
self.cluster = cluster
self.task = task
self.is_master = (task.type == "master" and task.index == 0)
self.train_dir = train_dir
self.config = tf.ConfigProto(log_device_placement=log_device_placement)
if self.is_master and self.task.index > 0:
raise StandardError("%s: Only one replica of master expected",
task_as_string(self.task))
def run(self, start_new_model=False):
"""Performs training on the currently defined Tensorflow graph.
Returns:
A tuple of the training Hit@1 and the training PERR.
"""
if self.is_master and start_new_model:
self.remove_training_directory(self.train_dir)
target, device_fn = self.start_server_if_distributed()
checkpoint = self.get_latest_checkpoint(start_new_model, self.train_dir)
with tf.Graph().as_default() as graph:
with tf.device(device_fn):
saver = self.build_model()
global_step = tf.get_collection("global_step")[0]
loss = tf.get_collection("loss")[0]
predictions = tf.get_collection("predictions")[0]
labels = tf.get_collection("labels")[0]
train_op = tf.get_collection("train_op")[0]
init_op = tf.global_variables_initializer()
if FLAGS.dropout:
keep_prob_tensor = tf.get_collection("keep_prob")[0]
if FLAGS.noise_level > 0:
noise_level_tensor = tf.get_collection("noise_level")[0]
if FLAGS.reweight:
weights_input, weights_assignment = None, None
if len(tf.get_collection("weights_input")) > 0:
weights_input = tf.get_collection("weights_input")[0]
weights_assignment = tf.get_collection("weights_assignment")[0]
sv = tf.train.Supervisor(
graph,
logdir=self.train_dir,
init_op=init_op,
is_chief=self.is_master,
global_step=global_step,
save_model_secs=15 * 60,
save_summaries_secs=120,
saver=saver)
logging.info("%s: Starting managed session.", task_as_string(self.task))
with sv.managed_session(target, config=self.config) as sess:
if checkpoint is not None:
saver.restore(sess, checkpoint)
# re-assign weights
if FLAGS.reweight:
optional_assign_weights(sess, weights_input, weights_assignment)
steps = 0
try:
logging.info("%s: Entering training loop.", task_as_string(self.task))
while not sv.should_stop():
steps += 1
batch_start_time = time.time()
custom_feed = {}
if FLAGS.dropout:
custom_feed[keep_prob_tensor] = FLAGS.keep_prob
if FLAGS.noise_level > 0:
custom_feed[noise_level_tensor] = FLAGS.noise_level
_, global_step_val, loss_val, predictions_val, labels_val = sess.run(
[train_op, global_step, loss, predictions, labels], feed_dict=custom_feed)
seconds_per_batch = time.time() - batch_start_time
if self.is_master:
examples_per_second = labels_val.shape[0] / seconds_per_batch
hit_at_one = eval_util.calculate_hit_at_one(predictions_val,
labels_val)
perr = eval_util.calculate_precision_at_equal_recall_rate(
predictions_val, labels_val)
recall = "N/A"
if False:
recall = eval_util.calculate_recall_at_n(
predictions_val, labels_val, FLAGS.recall_at_n)
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_Recall@%d" % FLAGS.recall_at_n, recall), global_step_val)
recall = "%.2f" % recall
gap = eval_util.calculate_gap(predictions_val, labels_val)
logging.info(
"%s: training step " + str(global_step_val) + "| Hit@1: " +
("%.2f" % hit_at_one) + " PERR: " + ("%.2f" % perr) + " GAP: " +
("%.2f" % gap) + " Recall@%d: " % FLAGS.recall_at_n +
recall + " Loss: " + str(loss_val),
task_as_string(self.task))
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_Hit@1", hit_at_one),
global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_Perr", perr), global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_GAP", gap), global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("global_step/Examples/Second",
examples_per_second), global_step_val)
sv.summary_writer.flush()
if FLAGS.max_steps is not None and steps > FLAGS.max_steps:
logging.info("%s: Done training -- max_steps limit reached.",
task_as_string(self.task))
break
except tf.errors.OutOfRangeError:
logging.info("%s: Done training -- epoch limit reached.",
task_as_string(self.task))
logging.info("%s: Exited training loop.", task_as_string(self.task))
sv.Stop()
def start_server_if_distributed(self):
"""Starts a server if the execution is distributed."""
if self.cluster:
logging.info("%s: Starting trainer within cluster %s.",
task_as_string(self.task), self.cluster.as_dict())
server = start_server(self.cluster, self.task)
target = server.target
device_fn = tf.train.replica_device_setter(
ps_device="/job:ps",
worker_device="/job:%s/task:%d" % (self.task.type, self.task.index),
cluster=self.cluster)
else:
target = ""
device_fn = ""
return (target, device_fn)
def remove_training_directory(self, train_dir):
"""Removes the training directory."""
try:
logging.info(
"%s: Removing existing train directory.",
task_as_string(self.task))
gfile.DeleteRecursively(train_dir)
except:
logging.error(
"%s: Failed to delete directory " + train_dir +
" when starting a new model. Please delete it manually and" +
" try again.", task_as_string(self.task))
def get_latest_checkpoint(self, start_new_model, train_dir):
if start_new_model:
logging.info("%s: Flag 'start_new_model' is set. Building a new model.",
task_as_string(self.task))
return None
latest_checkpoint = tf.train.latest_checkpoint(train_dir)
if not latest_checkpoint:
logging.info("%s: No checkpoint file found. Building a new model.",
task_as_string(self.task))
return None
return latest_checkpoint
def build_model(self):
"""Find the model and build the graph."""
# Convert feature_names and feature_sizes to lists of values.
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.distillation_features:
if FLAGS.frame_features:
reader = readers.YT8MFrameDistillationFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedDistillationFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
# Find the model.
model = find_class_by_name(FLAGS.model,
[frame_level_models, video_level_models])()
label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])()
optimizer_class = find_class_by_name(FLAGS.optimizer, [tf.train])
transformer_class = find_class_by_name(FLAGS.feature_transformer, [feature_transform])
augmenter_class = find_class_by_name(FLAGS.data_augmenter, [data_augmentation])
build_graph(reader=reader,
model=model,
optimizer_class=optimizer_class,
augmenter_class=augmenter_class,
transformer_class=transformer_class,
clip_gradient_norm=FLAGS.clip_gradient_norm,
train_data_pattern=FLAGS.train_data_pattern,
label_loss_fn=label_loss_fn,
base_learning_rate=FLAGS.base_learning_rate,
learning_rate_decay=FLAGS.learning_rate_decay,
learning_rate_decay_examples=FLAGS.learning_rate_decay_examples,
regularization_penalty=FLAGS.regularization_penalty,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size,
num_epochs=FLAGS.num_epochs)
logging.info("%s: Built graph.", task_as_string(self.task))
return tf.train.Saver(max_to_keep=3, keep_checkpoint_every_n_hours=FLAGS.keep_checkpoint_every_n_hours)
class ParameterServer(object):
"""A parameter server to serve variables in a distributed execution."""
def __init__(self, cluster, task):
"""Creates a ParameterServer.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
self.cluster = cluster
self.task = task
def run(self):
"""Starts the parameter server."""
logging.info("%s: Starting parameter server within cluster %s.",
task_as_string(self.task), self.cluster.as_dict())
server = start_server(self.cluster, self.task)
server.join()
def start_server(cluster, task):
"""Creates a Server.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
if not task.type:
raise ValueError("%s: The task type must be specified." %
task_as_string(task))
if task.index is None:
raise ValueError("%s: The task index must be specified." %
task_as_string(task))
# Create and start a server.
return tf.train.Server(
tf.train.ClusterSpec(cluster),
protocol="grpc",
job_name=task.type,
task_index=task.index)
def task_as_string(task):
return "/job:%s/task:%s" % (task.type, task.index)
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object,), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.",
task_as_string(task), tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
Trainer(cluster, task, FLAGS.train_dir, FLAGS.log_device_placement).run(
start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-wangheda/train.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for training Tensorflow models on the YouTube-8M dataset."""
import json
import os
import time
import numpy
import eval_util
import losses
import frame_level_models
import video_level_models
import data_augmentation
import feature_transform
import readers
import tensorflow as tf
import tensorflow.contrib.slim as slim
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import utils
FLAGS = flags.FLAGS
if __name__ == "__main__":
# Dataset flags.
flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
"The directory to save the model files in.")
flags.DEFINE_string(
"train_data_pattern", "",
"File glob for the training dataset. If the files refer to Frame Level "
"features (i.e. tensorflow.SequenceExample), then set --reader_type "
"format. The (Sequence)Examples are expected to have 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024", "Length of the feature vectors.")
# Distillation flags
flags.DEFINE_bool(
"distillation_features", False,
"If set, *DistillationFeatureReader will be used, the feature must contains"
"prediction features (shape = [4716]).")
flags.DEFINE_integer(
"distillation_type", 0, "Type of distillation, options are 1 and 2.")
flags.DEFINE_bool(
"distillation_as_input", False, "If set true, distillation_predictions will be given to model.")
flags.DEFINE_bool(
"distillation_as_boosting", False, "If set true, distillation_predictions will be used in computation of weighted loss.")
flags.DEFINE_float("distillation_percent", 0.0,
"If larger than 0, final_loss = distillation_loss * percent + normal_loss * (1.0 - percent).")
# Model flags.
flags.DEFINE_bool(
"frame_features", False,
"If set, then --train_data_pattern must be frame-level features. "
"Otherwise, --train_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_string(
"model", "LogisticModel",
"Which architecture to use for the model. Models are defined "
"in models.py.")
flags.DEFINE_bool(
"multitask", False,
"Whether to consider support_predictions")
flags.DEFINE_bool(
"start_new_model", False,
"If set, this will not resume from a checkpoint and will instead create a"
" new model instance.")
# Training flags.
flags.DEFINE_integer("batch_size", 1024,
"How many examples to process per batch for training.")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Which loss function to use for training the model.")
flags.DEFINE_float(
"regularization_penalty", 1,
"How much weight to give to the regularization loss (the label loss has "
"a weight of 1).")
flags.DEFINE_float("base_learning_rate", 0.01,
"Which learning rate to start with.")
flags.DEFINE_float("learning_rate_decay", 0.95,
"Learning rate decay factor to be applied every "
"learning_rate_decay_examples.")
flags.DEFINE_float("learning_rate_decay_examples", 4000000,
"Multiply current learning rate by learning_rate_decay "
"every learning_rate_decay_examples.")
flags.DEFINE_integer("num_epochs", 5,
"How many passes to make over the dataset before "
"halting training.")
flags.DEFINE_integer("max_steps", None,
"How many steps before stop.")
flags.DEFINE_float("keep_checkpoint_every_n_hours", 1.0,
"How many hours before saving a new checkpoint")
flags.DEFINE_integer("keep_checkpoint_interval", 15,
"How many minutes before saving a new checkpoint")
flags.DEFINE_bool("reweight", False,
"Whether to load model weight from file.")
flags.DEFINE_string("sample_vocab_file", "",
"Where to load video_id vocabulary.")
flags.DEFINE_string("sample_freq_file", "",
"Where to load sample frequency.")
# Other flags.
flags.DEFINE_integer("num_readers", 8,
"How many threads to use for reading input files.")
flags.DEFINE_string("optimizer", "AdamOptimizer",
"What optimizer class to use.")
flags.DEFINE_float("clip_gradient_norm", 1.0, "Norm to clip gradients to.")
flags.DEFINE_bool(
"log_device_placement", False,
"Whether to write the device on which every op will run into the "
"logs on startup.")
flags.DEFINE_integer("recall_at_n", 100,
"N in recall@N.")
flags.DEFINE_bool(
"dropout", False,
"Whether to consider dropout")
flags.DEFINE_float("keep_prob", 1.0,
"probability to keep output (used in dropout, keep it unchanged in validationg and test)")
flags.DEFINE_float("noise_level", 0.0,
"standard deviation of noise (added to hidden nodes)")
def validate_class_name(flag_value, category, modules, expected_superclass):
"""Checks that the given string matches a class of the expected type.
Args:
flag_value: A string naming the class to instantiate.
category: A string used further describe the class in error messages
(e.g. 'model', 'reader', 'loss').
modules: A list of modules to search for the given class.
expected_superclass: A class that the given class should inherit from.
Raises:
FlagsError: If the given class could not be found or if the first class
found with that name doesn't inherit from the expected superclass.
Returns:
True if a class was found that matches the given constraints.
"""
candidates = [getattr(module, flag_value, None) for module in modules]
for candidate in candidates:
if not candidate:
continue
if not issubclass(candidate, expected_superclass):
raise flags.FlagsError("%s '%s' doesn't inherit from %s." %
(category, flag_value,
expected_superclass.__name__))
return True
raise flags.FlagsError("Unable to find %s '%s'." % (category, flag_value))
def get_input_data_tensors(reader,
data_pattern,
batch_size=1000,
num_epochs=None,
num_readers=1):
"""Creates the section of the graph which reads the training data.
Args:
reader: A class which parses the training data.
data_pattern: A 'glob' style path to the data files.
batch_size: How many examples to process at a time.
num_epochs: How many passes to make over the training data. Set to 'None'
to run indefinitely.
num_readers: How many I/O threads to use.
Returns:
A tuple containing the features tensor, labels tensor, and optionally a
tensor containing the number of frames per video. The exact dimensions
depend on the reader being used.
Raises:
IOError: If no files matching the given pattern were found.
"""
logging.info("Using batch size of " + str(batch_size) + " for training.")
with tf.name_scope("train_input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find training files. data_pattern='" +
data_pattern + "'.")
logging.info("Number of training files: %s.", str(len(files)))
filename_queue = tf.train.string_input_producer(
files, num_epochs=num_epochs, shuffle=True)
training_data = [
reader.prepare_reader(filename_queue) for _ in range(num_readers)
]
return tf.train.shuffle_batch_join(
training_data,
batch_size=batch_size,
capacity=FLAGS.batch_size * 10,
min_after_dequeue=FLAGS.batch_size,
allow_smaller_final_batch=True,
enqueue_many=True)
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def get_video_weights_array():
weight_lines = open(FLAGS.sample_freq_file).readlines()
weights = numpy.array(map(float, weight_lines))
weights = weights.reshape([len(weight_lines)])
return weights, len(weight_lines)
def optional_assign_weights(sess, weights_input, weights_assignment):
if weights_input is not None:
weights, length = get_video_weights_array()
_ = sess.run(weights_assignment, feed_dict={weights_input: weights})
print "Assigned weights from %s" % FLAGS.sample_freq_file
else:
print "Collection weights_input not found"
def get_video_weights(video_id_batch):
video_id_to_index = tf.contrib.lookup.string_to_index_table_from_file(
vocabulary_file=FLAGS.sample_vocab_file, default_value=0)
indexes = video_id_to_index.lookup(video_id_batch)
weights, length = get_video_weights_array()
weights_input = tf.placeholder(tf.float32, shape=[length], name="sample_weights_input")
weights_tensor = tf.get_variable("sample_weights",
shape=[length],
trainable=False,
dtype=tf.float32,
initializer=tf.constant_initializer(weights))
weights_assignment = tf.assign(weights_tensor, weights_input)
tf.add_to_collection("weights_input", weights_input)
tf.add_to_collection("weights_assignment", weights_assignment)
video_weight_batch = tf.nn.embedding_lookup(weights_tensor, indexes)
return video_weight_batch
def get_weights_by_predictions(labels_batch, predictions):
epsilon = 1e-6
float_labels = tf.cast(labels_batch, dtype=tf.float32)
cross_entropy_loss = float_labels * tf.log(predictions + epsilon) + (
1 - float_labels) * tf.log(1 - predictions + epsilon)
ce = tf.reduce_sum(tf.negative(cross_entropy_loss), axis=1)
mean_ce = tf.reduce_mean(ce + epsilon)
weights = tf.where(ce > mean_ce,
3.0 * tf.ones_like(ce),
0.5 * tf.ones_like(ce))
return weights
def build_graph(reader,
model,
train_data_pattern,
label_loss_fn=losses.CrossEntropyLoss(),
batch_size=1000,
base_learning_rate=0.01,
learning_rate_decay_examples=1000000,
learning_rate_decay=0.95,
optimizer_class=tf.train.AdamOptimizer,
transformer_class=feature_transform.DefaultTransformer,
augmenter_class=data_augmentation.DefaultAugmenter,
clip_gradient_norm=1.0,
regularization_penalty=1,
num_readers=1,
num_epochs=None):
"""Creates the Tensorflow graph.
This will only be called once in the life of
a training model, because after the graph is created the model will be
restored from a meta graph file rather than being recreated.
Args:
reader: The data file reader. It should inherit from BaseReader.
model: The core model (e.g. logistic or neural net). It should inherit
from BaseModel.
train_data_pattern: glob path to the training data files.
label_loss_fn: What kind of loss to apply to the model. It should inherit
from BaseLoss.
batch_size: How many examples to process at a time.
base_learning_rate: What learning rate to initialize the optimizer with.
optimizer_class: Which optimization algorithm to use.
clip_gradient_norm: Magnitude of the gradient to clip to.
regularization_penalty: How much weight to give the regularization loss
compared to the label loss.
num_readers: How many threads to use for I/O operations.
num_epochs: How many passes to make over the data. 'None' means an
unlimited number of passes.
"""
global_step = tf.Variable(0, trainable=False, name="global_step")
learning_rate = tf.train.exponential_decay(
base_learning_rate,
global_step * batch_size,
learning_rate_decay_examples,
learning_rate_decay,
staircase=True)
tf.summary.scalar('learning_rate', learning_rate)
optimizer = optimizer_class(learning_rate)
if FLAGS.distillation_features:
video_id, model_input_raw, labels_batch, num_frames, distill_labels_batch = (
get_input_data_tensors(
reader,
train_data_pattern,
batch_size=batch_size,
num_readers=num_readers,
num_epochs=num_epochs))
if FLAGS.distillation_features and FLAGS.distillation_type == 2:
p = FLAGS.distillation_percent
print "distillation_percent =", p, "reforming labels"
float_labels = tf.cast(labels_batch, dtype=tf.float32)
sum_float_labels = tf.reduce_sum(float_labels, axis=1, keep_dims=True)
sum_distill_labels = tf.reduce_sum(distill_labels_batch, axis=1, keep_dims=True) + 1e-6
distill_labels_batch = float_labels + distill_labels_batch * (sum_float_labels / sum_distill_labels * p)
distill_labels_batch = tf.clip_by_value(distill_labels_batch, clip_value_min=0.0, clip_value_max=1.0)
else:
video_id, model_input_raw, labels_batch, num_frames = (
get_input_data_tensors(
reader,
train_data_pattern,
batch_size=batch_size,
num_readers=num_readers,
num_epochs=num_epochs))
# data augmentation, will not persist in inference
data_augmenter = augmenter_class()
model_input_raw, labels_batch, num_frames = data_augmenter.augment(model_input_raw, num_frames=num_frames, labels_batch=labels_batch)
tf.summary.histogram("model/input_raw", model_input_raw)
feature_transformer = transformer_class()
model_input, num_frames = feature_transformer.transform(model_input_raw, num_frames=num_frames)
tf.summary.histogram("model/input", model_input)
with tf.name_scope("model"):
if FLAGS.noise_level > 0:
noise_level_tensor = tf.placeholder_with_default(0.0, shape=[], name="noise_level")
else:
noise_level_tensor = None
if FLAGS.distillation_as_input:
distillation_predictions = distill_labels_batch
else:
distillation_predictions = None
if FLAGS.dropout:
keep_prob_tensor = tf.placeholder_with_default(1.0, shape=[], name="keep_prob")
result = model.create_model(
model_input,
num_frames=num_frames,
vocab_size=reader.num_classes,
labels=labels_batch,
dropout=FLAGS.dropout,
keep_prob=keep_prob_tensor,
distillation_predictions=distillation_predictions,
noise_level=noise_level_tensor)
else:
result = model.create_model(
model_input,
num_frames=num_frames,
vocab_size=reader.num_classes,
labels=labels_batch,
distillation_predictions=distillation_predictions,
noise_level=noise_level_tensor)
for variable in slim.get_model_variables():
tf.summary.histogram(variable.op.name, variable)
print "result", result
predictions = result["predictions"]
if "loss" in result.keys():
label_loss = result["loss"]
else:
video_weights_batch = None
if FLAGS.reweight:
video_weights_batch = get_video_weights(video_id)
if FLAGS.distillation_as_boosting:
video_weights_batch = get_weights_by_predictions(labels_batch, distillation_predictions)
if FLAGS.multitask:
support_predictions = result["support_predictions"]
tf.summary.histogram("model/support_predictions", support_predictions)
print "support_predictions", support_predictions
if FLAGS.distillation_features and FLAGS.distillation_type == 1:
p = FLAGS.distillation_percent
print "distillation_percent =", p
if p <= 0:
label_loss = label_loss_fn.calculate_loss(predictions, support_predictions, labels_batch, weights=video_weights_batch)
elif p >= 1:
label_loss = label_loss_fn.calculate_loss(predictions, support_predictions, distill_labels_batch, weights=video_weights_batch)
else:
label_loss = label_loss_fn.calculate_loss(predictions, support_predictions, labels_batch, weights=video_weights_batch) * (1.0 - p) \
+ label_loss_fn.calculate_loss(predictions, support_predictions, distill_labels_batch, weights=video_weights_batch) * p
elif FLAGS.distillation_features and FLAGS.distillation_type == 2:
print "using pure distillation loss"
label_loss = label_loss_fn.calculate_loss(predictions, support_predictions, distill_labels_batch, weights=video_weights_batch)
else:
print "using original loss"
label_loss = label_loss_fn.calculate_loss(predictions, support_predictions, labels_batch, weights=video_weights_batch)
else:
if FLAGS.distillation_features and FLAGS.distillation_type == 1:
p = FLAGS.distillation_percent
print "distillation_percent =", p
if p <= 0:
label_loss = label_loss_fn.calculate_loss(predictions, labels_batch, weights=video_weights_batch)
elif p >= 1:
label_loss = label_loss_fn.calculate_loss(predictions, distill_labels_batch, weights=video_weights_batch)
else:
label_loss = label_loss_fn.calculate_loss(predictions, labels_batch, weights=video_weights_batch) * (1.0 - p) \
+ label_loss_fn.calculate_loss(predictions, distill_labels_batch, weights=video_weights_batch) * p
elif FLAGS.distillation_features and FLAGS.distillation_type == 2:
print "using pure distillation loss"
label_loss = label_loss_fn.calculate_loss(predictions, distill_labels_batch, weights=video_weights_batch)
else:
print "using original loss"
label_loss = label_loss_fn.calculate_loss(predictions, labels_batch, weights=video_weights_batch)
tf.summary.histogram("model/predictions", predictions)
tf.summary.scalar("label_loss", label_loss)
if "regularization_loss" in result.keys():
reg_loss = result["regularization_loss"]
else:
reg_loss = tf.constant(0.0)
reg_losses = tf.losses.get_regularization_losses()
if reg_losses:
reg_loss += tf.add_n(reg_losses)
if regularization_penalty != 0:
tf.summary.scalar("reg_loss", reg_loss)
# Adds update_ops (e.g., moving average updates in batch normalization) as
# a dependency to the train_op.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
if "update_ops" in result.keys():
update_ops += result["update_ops"]
if update_ops:
with tf.control_dependencies(update_ops):
barrier = tf.no_op(name="gradient_barrier")
with tf.control_dependencies([barrier]):
label_loss = tf.identity(label_loss)
# Incorporate the L2 weight penalties etc.
final_loss = regularization_penalty * reg_loss + label_loss
gradients = optimizer.compute_gradients(final_loss,
colocate_gradients_with_ops=False)
if clip_gradient_norm > 0:
with tf.name_scope('clip_grads'):
gradients = utils.clip_gradient_norms(gradients , clip_gradient_norm)
train_op = optimizer.apply_gradients(gradients, global_step=global_step)
tf.add_to_collection("global_step", global_step)
tf.add_to_collection("loss", label_loss)
tf.add_to_collection("predictions", predictions)
tf.add_to_collection("input_batch_raw", model_input_raw)
tf.add_to_collection("input_batch", model_input)
tf.add_to_collection("num_frames", num_frames)
tf.add_to_collection("labels", tf.cast(labels_batch, tf.float32))
tf.add_to_collection("train_op", train_op)
if FLAGS.dropout:
tf.add_to_collection("keep_prob", keep_prob_tensor)
if FLAGS.noise_level > 0:
tf.add_to_collection("noise_level", noise_level_tensor)
class Trainer(object):
"""A Trainer to train a Tensorflow graph."""
def __init__(self, cluster, task, train_dir, log_device_placement=True):
""""Creates a Trainer.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
self.cluster = cluster
self.task = task
self.is_master = (task.type == "master" and task.index == 0)
self.train_dir = train_dir
self.config = tf.ConfigProto(log_device_placement=log_device_placement)
if self.is_master and self.task.index > 0:
raise StandardError("%s: Only one replica of master expected",
task_as_string(self.task))
def run(self, start_new_model=False):
"""Performs training on the currently defined Tensorflow graph.
Returns:
A tuple of the training Hit@1 and the training PERR.
"""
if self.is_master and start_new_model:
self.remove_training_directory(self.train_dir)
target, device_fn = self.start_server_if_distributed()
meta_filename = self.get_meta_filename(start_new_model, self.train_dir)
with tf.Graph().as_default() as graph:
if meta_filename:
saver = self.recover_model(meta_filename)
with tf.device(device_fn):
if not meta_filename:
saver = self.build_model()
global_step = tf.get_collection("global_step")[0]
loss = tf.get_collection("loss")[0]
predictions = tf.get_collection("predictions")[0]
labels = tf.get_collection("labels")[0]
train_op = tf.get_collection("train_op")[0]
init_op = tf.global_variables_initializer()
if FLAGS.dropout:
keep_prob_tensor = tf.get_collection("keep_prob")[0]
if FLAGS.noise_level > 0:
noise_level_tensor = tf.get_collection("noise_level")[0]
if FLAGS.reweight:
weights_input, weights_assignment = None, None
if len(tf.get_collection("weights_input")) > 0:
weights_input = tf.get_collection("weights_input")[0]
weights_assignment = tf.get_collection("weights_assignment")[0]
sv = tf.train.Supervisor(
graph,
logdir=self.train_dir,
init_op=init_op,
is_chief=self.is_master,
global_step=global_step,
save_model_secs=FLAGS.keep_checkpoint_interval * 60,
save_summaries_secs=120,
saver=saver)
logging.info("%s: Starting managed session.", task_as_string(self.task))
with sv.managed_session(target, config=self.config) as sess:
# re-assign weights
if FLAGS.reweight:
optional_assign_weights(sess, weights_input, weights_assignment)
steps = 0
try:
logging.info("%s: Entering training loop.", task_as_string(self.task))
while not sv.should_stop():
steps += 1
batch_start_time = time.time()
custom_feed = {}
if FLAGS.dropout:
custom_feed[keep_prob_tensor] = FLAGS.keep_prob
if FLAGS.noise_level > 0:
custom_feed[noise_level_tensor] = FLAGS.noise_level
_, global_step_val, loss_val, predictions_val, labels_val = sess.run(
[train_op, global_step, loss, predictions, labels], feed_dict=custom_feed)
seconds_per_batch = time.time() - batch_start_time
if self.is_master:
examples_per_second = labels_val.shape[0] / seconds_per_batch
hit_at_one = eval_util.calculate_hit_at_one(predictions_val,
labels_val)
perr = eval_util.calculate_precision_at_equal_recall_rate(
predictions_val, labels_val)
recall = "N/A"
if False:
recall = eval_util.calculate_recall_at_n(
predictions_val, labels_val, FLAGS.recall_at_n)
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_Recall@%d" % FLAGS.recall_at_n, recall), global_step_val)
recall = "%.2f" % recall
gap = eval_util.calculate_gap(predictions_val, labels_val)
logging.info(
"%s: training step " + str(global_step_val) + "| Hit@1: " +
("%.2f" % hit_at_one) + " PERR: " + ("%.2f" % perr) + " GAP: " +
("%.2f" % gap) + " Recall@%d: " % FLAGS.recall_at_n +
recall + " Loss: " + str(loss_val),
task_as_string(self.task))
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_Hit@1", hit_at_one),
global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_Perr", perr), global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_GAP", gap), global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("global_step/Examples/Second",
examples_per_second), global_step_val)
sv.summary_writer.flush()
if FLAGS.max_steps is not None and steps > FLAGS.max_steps:
logging.info("%s: Done training -- max_steps limit reached.",
task_as_string(self.task))
break
except tf.errors.OutOfRangeError:
logging.info("%s: Done training -- epoch limit reached.",
task_as_string(self.task))
logging.info("%s: Exited training loop.", task_as_string(self.task))
sv.Stop()
def start_server_if_distributed(self):
"""Starts a server if the execution is distributed."""
if self.cluster:
logging.info("%s: Starting trainer within cluster %s.",
task_as_string(self.task), self.cluster.as_dict())
server = start_server(self.cluster, self.task)
target = server.target
device_fn = tf.train.replica_device_setter(
ps_device="/job:ps",
worker_device="/job:%s/task:%d" % (self.task.type, self.task.index),
cluster=self.cluster)
else:
target = ""
device_fn = ""
return (target, device_fn)
def remove_training_directory(self, train_dir):
"""Removes the training directory."""
try:
logging.info(
"%s: Removing existing train directory.",
task_as_string(self.task))
gfile.DeleteRecursively(train_dir)
except:
logging.error(
"%s: Failed to delete directory " + train_dir +
" when starting a new model. Please delete it manually and" +
" try again.", task_as_string(self.task))
def get_meta_filename(self, start_new_model, train_dir):
if start_new_model:
logging.info("%s: Flag 'start_new_model' is set. Building a new model.",
task_as_string(self.task))
return None
latest_checkpoint = tf.train.latest_checkpoint(train_dir)
if not latest_checkpoint:
logging.info("%s: No checkpoint file found. Building a new model.",
task_as_string(self.task))
return None
meta_filename = latest_checkpoint + ".meta"
if not gfile.Exists(meta_filename):
logging.info("%s: No meta graph file found. Building a new model.",
task_as_string(self.task))
return None
else:
return meta_filename
def recover_model(self, meta_filename):
logging.info("%s: Restoring from meta graph file %s",
task_as_string(self.task), meta_filename)
return tf.train.import_meta_graph(meta_filename)
def build_model(self):
"""Find the model and build the graph."""
# Convert feature_names and feature_sizes to lists of values.
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.distillation_features:
print "distillation readers"
if FLAGS.frame_features:
reader = readers.YT8MFrameDistillationFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedDistillationFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
# Find the model.
model = find_class_by_name(FLAGS.model,
[frame_level_models, video_level_models])()
label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])()
optimizer_class = find_class_by_name(FLAGS.optimizer, [tf.train])
transformer_class = find_class_by_name(FLAGS.feature_transformer, [feature_transform])
augmenter_class = find_class_by_name(FLAGS.data_augmenter, [data_augmentation])
build_graph(reader=reader,
model=model,
optimizer_class=optimizer_class,
augmenter_class=augmenter_class,
transformer_class=transformer_class,
clip_gradient_norm=FLAGS.clip_gradient_norm,
train_data_pattern=FLAGS.train_data_pattern,
label_loss_fn=label_loss_fn,
base_learning_rate=FLAGS.base_learning_rate,
learning_rate_decay=FLAGS.learning_rate_decay,
learning_rate_decay_examples=FLAGS.learning_rate_decay_examples,
regularization_penalty=FLAGS.regularization_penalty,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size,
num_epochs=FLAGS.num_epochs)
logging.info("%s: Built graph.", task_as_string(self.task))
return tf.train.Saver(max_to_keep=3, keep_checkpoint_every_n_hours=FLAGS.keep_checkpoint_every_n_hours)
class ParameterServer(object):
"""A parameter server to serve variables in a distributed execution."""
def __init__(self, cluster, task):
"""Creates a ParameterServer.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
self.cluster = cluster
self.task = task
def run(self):
"""Starts the parameter server."""
logging.info("%s: Starting parameter server within cluster %s.",
task_as_string(self.task), self.cluster.as_dict())
server = start_server(self.cluster, self.task)
server.join()
def start_server(cluster, task):
"""Creates a Server.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
if not task.type:
raise ValueError("%s: The task type must be specified." %
task_as_string(task))
if task.index is None:
raise ValueError("%s: The task index must be specified." %
task_as_string(task))
# Create and start a server.
return tf.train.Server(
tf.train.ClusterSpec(cluster),
protocol="grpc",
job_name=task.type,
task_index=task.index)
def task_as_string(task):
return "/job:%s/task:%s" % (task.type, task.index)
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object,), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.",
task_as_string(task), tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
Trainer(cluster, task, FLAGS.train_dir, FLAGS.log_device_placement).run(
start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-wangheda/training_scripts/run-cascade-75-chaining-cnn.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/distillchain_cnn_dcc" \
--train_data_pattern="/Youtube-8M/distillation/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_as_input=True \
--model=DistillchainCnnDeepCombineChainModel \
--deep_chain_layers=3 \
--deep_chain_relu_cells=256 \
--moe_num_mixtures=4 \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_type="label,label,label" \
--support_loss_percent=0.05 \
--num_readers=4 \
--batch_size=128 \
--num_epochs=3 \
--keep_checkpoint_every_n_hours=0.5 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-wangheda/training_scripts/run-cascade-75-chaining-lstm-cnn.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/distillchain_lstmcnn_dcc" \
--train_data_pattern="/Youtube-8M/distillation/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_as_input=True \
--model=DistillchainLstmCnnDeepCombineChainModel \
--deep_chain_layers=2 \
--deep_chain_relu_cells=128 \
--distillchain_relu_cells=256 \
--moe_num_mixtures=4 \
--lstm_layers=1 \
--lstm_cells="1024,128" \
--rnn_swap_memory=True \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_type="label,label" \
--support_loss_percent=0.1 \
--num_readers=4 \
--batch_size=128 \
--num_epochs=3 \
--keep_checkpoint_every_n_hours=1.0 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-wangheda/training_scripts/run-cascade-75-chaining-lstm.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/distillchain_multilstm_dcc" \
--train_data_pattern="/Youtube-8M/distillation/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_as_input=True \
--model=DistillchainLstmMemoryDeepCombineChainModel \
--lstm_layers=1 \
--lstm_cells=1024 \
--moe_num_mixtures=4 \
--distillation_relu_cells=256 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=2 \
--rnn_swap_memory=True \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_type="label,label" \
--support_loss_percent=0.1 \
--num_readers=4 \
--batch_size=128 \
--num_epochs=3 \
--keep_checkpoint_every_n_hours=1.0 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-wangheda/training_scripts/run-cascade-75-chaining-parallel-lstm.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/distillchain_lstmparalleloutput" \
--train_data_pattern="/Youtube-8M/distillation/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_as_input=True \
--model=DistillchainLstmParallelFinaloutputModel \
--rnn_swap_memory=True \
--lstm_cells="1024,128" \
--moe_num_mixtures=8 \
--num_readers=4 \
--batch_size=128 \
--num_epochs=3 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-wangheda/training_scripts/run-cascade-75-chaining-video.sh
================================================
noise_level=0.15
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/distillchain_video_dcc" \
--train_data_pattern="/Youtube-8M/distillation/video/train/*.tfrecord" \
--distillation_features=True \
--distillation_as_input=True \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=DistillchainDeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_layers=4 \
--deep_chain_relu_cells=256 \
--data_augmenter=NoiseAugmenter \
--input_noise_level=$noise_level \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_type="label,label,label,label" \
--num_supports=18864 \
--support_loss_percent=0.05 \
--base_learning_rate=0.006 \
--keep_checkpoint_every_n_hour=0.25 \
--num_readers=5 \
--num_epochs=6 \
--batch_size=512
================================================
FILE: youtube-8m-wangheda/training_scripts/run-cascade-75-multiple-attention-pooling.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/distillchain_lstmattention8max" \
--train_data_pattern="/Youtube-8M/distillation/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_as_input=True \
--model=DistillchainLstmAttentionMaxPoolingModel \
--moe_num_mixtures=8 \
--lstm_attentions=8 \
--lstm_cells=1024 \
--rnn_swap_memory=True \
--moe_num_mixtures=8 \
--num_readers=4 \
--batch_size=128 \
--num_epochs=3 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-wangheda/training_scripts/run-cascade-76-chaining-cnn.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/distillchain_v2_cnn_dcc" \
--train_data_pattern="/Youtube-8M/distillation_v2/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_as_input=True \
--model=DistillchainCnnDeepCombineChainModel \
--deep_chain_layers=3 \
--deep_chain_relu_cells=256 \
--moe_num_mixtures=4 \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_type="label,label,label" \
--support_loss_percent=0.05 \
--num_readers=4 \
--batch_size=128 \
--num_epochs=3 \
--keep_checkpoint_every_n_hours=0.5 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-wangheda/training_scripts/run-cascade-76-chaining-lstm-cnn.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/distillchain_v2_lstmcnn_dcc" \
--train_data_pattern="/Youtube-8M/distillation_v2/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_as_input=True \
--model=DistillchainLstmCnnDeepCombineChainModel \
--deep_chain_layers=2 \
--deep_chain_relu_cells=128 \
--distillchain_relu_cells=256 \
--moe_num_mixtures=4 \
--lstm_layers=1 \
--lstm_cells="1024,128" \
--rnn_swap_memory=True \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_type="label,label" \
--support_loss_percent=0.1 \
--num_readers=4 \
--batch_size=128 \
--num_epochs=3 \
--keep_checkpoint_every_n_hours=1.0 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-wangheda/training_scripts/run-cascade-76-chaining-lstm.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train-with-predictions.py \
--train_dir="../model/distillchain_v2_multilstm_dcc" \
--train_data_pattern="/Youtube-8M/data/frame/train/*.tfrecord" \
--predictions_data_pattern="/Youtube-8M/distillation_v2/predictions/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=False \
--distillation_as_input=True \
--model=DistillchainLstmMemoryDeepCombineChainModel \
--lstm_layers=1 \
--lstm_cells=1024 \
--moe_num_mixtures=4 \
--distillation_relu_cells=256 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=2 \
--rnn_swap_memory=True \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_type="label,label" \
--support_loss_percent=0.1 \
--num_readers=4 \
--batch_size=128 \
--num_epochs=2 \
--keep_checkpoint_every_n_hours=2.0 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-wangheda/training_scripts/run-cascade-76-chaining-video.sh
================================================
noise_level=0.15
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/distillchain_v2_video_dcc" \
--train_data_pattern="/Youtube-8M/distillation_v2/video/train/*.tfrecord" \
--distillation_features=True \
--distillation_as_input=True \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=DistillchainDeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_layers=4 \
--deep_chain_relu_cells=256 \
--data_augmenter=NoiseAugmenter \
--input_noise_level=$noise_level \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_type="label,label,label,label" \
--num_supports=18864 \
--support_loss_percent=0.05 \
--base_learning_rate=0.007 \
--keep_checkpoint_every_n_hour=0.25 \
--num_readers=5 \
--num_epochs=3 \
--batch_size=512
================================================
FILE: youtube-8m-wangheda/training_scripts/run-cascade-76-multiple-attention-pooling.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/distillchain_v2_lstmattention8max" \
--train_data_pattern="/Youtube-8M/distillation_v2/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_as_input=True \
--model=DistillchainLstmAttentionMaxPoolingModel \
--moe_num_mixtures=8 \
--lstm_attentions=8 \
--lstm_cells=1024 \
--rnn_swap_memory=True \
--moe_num_mixtures=8 \
--num_readers=4 \
--batch_size=128 \
--num_epochs=3 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-wangheda/training_scripts/run-cascade-76-multiscale-cnn-lstm.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/distillchain_v2_multiscale_cnnlstm" \
--train_data_pattern="/Youtube-8M/distillation_v2/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_as_input=True \
--model=DistillchainMultiscaleCnnLstmModel \
--multiscale_cnn_lstm_layers=3 \
--moe_num_mixtures=4 \
--rnn_swap_memory=True \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_loss_percent=1.0 \
--support_type="label,label,label" \
--is_training=True \
--num_readers=4 \
--batch_size=128 \
--num_epochs=3 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-wangheda/training_scripts/run-cascade-76-parallel-lstm-boosting.sh
================================================
CUDA_VISIBLE_DEVICES=1 python train.py \
--train_dir="../model/distillchain_v2_boost_lstmparalleloutput" \
--train_data_pattern="/Youtube-8M/distillation_v2/frame/train/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=True \
--distillation_as_input=True \
--distillation_as_boosting=True \
--model=DistillchainLstmParallelFinaloutputModel \
--rnn_swap_memory=True \
--lstm_layers=1 \
--lstm_cells="1024,128" \
--moe_num_mixtures=8 \
--num_readers=4 \
--batch_size=128 \
--num_epochs=3 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-wangheda/training_scripts/run-cascade-76-parallel-lstm.sh
================================================
CUDA_VISIBLE_DEVICES=1 python train-with-predictions.py \
--train_dir="../model/distillchain_v2_lstmparalleloutput" \
--train_data_pattern="/Youtube-8M/data/frame/train/*.tfrecord" \
--predictions_data_pattern="/Youtube-8M/distillation_v2/predictions/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--distillation_features=False \
--distillation_as_input=True \
--model=DistillchainLstmParallelFinaloutputModel \
--rnn_swap_memory=True \
--lstm_cells="1024,128" \
--moe_num_mixtures=4 \
--num_readers=4 \
--batch_size=128 \
--num_epochs=2 \
--keep_checkpoint_every_n_hours=2.0 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-wangheda/training_scripts/run-chaining-cnn.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/cnn_deep_combine_chain/" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=CnnDeepCombineChainModel \
--deep_chain_layers=4 \
--deep_chain_relu_cells=128 \
--moe_num_mixtures=4 \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_type="label,label,label,label" \
--support_loss_percent=0.05 \
--num_readers=4 \
--batch_size=128 \
--keep_checkpoint_every_n_hours=0.5 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-wangheda/training_scripts/run-chaining-deep-cnn.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/deep_cnn_deep_combine_chain/" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=DeepCnnDeepCombineChainModel \
--deep_chain_layers=2 \
--deep_chain_relu_cells=256 \
--deep_cnn_base_size=128 \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label" \
--support_loss_percent=0.1 \
--moe_num_mixtures=4 \
--batch_size=128 \
--dropout=True \
--keep_prob=1.0 \
--num_readers=4 \
--num_epochs=2 \
--keep_checkpoint_every_n_hours=0.5 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-wangheda/training_scripts/run-chaining-lstm-cnn.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/lstm_cnn_deep_combine_chain/" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=LstmCnnDeepCombineChainModel \
--deep_chain_layers=3 \
--deep_chain_relu_cells=128 \
--moe_num_mixtures=4 \
--lstm_layers=1 \
--lstm_cells="1024,128" \
--rnn_swap_memory=True \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_type="label,label,label" \
--support_loss_percent=0.05 \
--keep_checkpoint_every_n_hours=1.0 \
--batch_size=96 \
--num_readers=4 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-wangheda/training_scripts/run-chaining-lstm.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/multilstmmemory1024_moe4_deep_chain/" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--model=LstmMemoryDeepChainModel \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=200 \
--deep_chain_layers=1 \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--lstm_layers=2 \
--lstm_cells=1024 \
--multitask=True \
--support_type="label" \
--num_supports=4716 \
--label_loss=MultiTaskCrossEntropyLoss \
--base_learning_rate=0.0008 \
--support_loss_percent=0.2 \
--keep_checkpoint_every_n_hours=1.75 \
--rnn_swap_memory=True \
--batch_size=128
================================================
FILE: youtube-8m-wangheda/training_scripts/run-chaining-multi-resolution-lstm.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/multires_lstm_deep_combine_chain/" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label,label,label,label" \
--support_loss_percent=0.05 \
--support_size=18864 \
--model=MultiresLstmMemoryDeepCombineChainModel \
--deep_chain_layers=4 \
--deep_chain_relu_cells=256 \
--moe_num_mixtures=4 \
--dropout=True \
--keep_prob=0.9 \
--keep_checkpoint_every_n_hours=2.0 \
--base_learning_rate=0.0008 \
--feature_transformer=IdenticalTransformer \
--num_readers=4 \
--batch_size=128 \
--num_epochs=5 \
--rnn_swap_memory=True \
--lstm_cells="512,64" \
--lstm_layers=2
================================================
FILE: youtube-8m-wangheda/training_scripts/run-chaining-shared-lstm.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/lstmmem1024_deep_combine_chain_length" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=LstmMemoryModel \
--video_level_classifier_model=DeepCombineChainModel \
--moe_num_mixtures=4 \
--deep_chain_relu_cells=256 \
--deep_chain_layers=1 \
--deep_chain_use_length=True \
--label_loss=MultiTaskCrossEntropyLoss \
--multitask=True \
--support_type="label" \
--num_supports=4716 \
--support_loss_percent=0.05 \
--keep_checkpoint_every_n_hour=1.0 \
--base_learning_rate=0.0005 \
--num_readers=8 \
--num_epochs=5 \
--rnn_swap_memory=True \
--batch_size=128
================================================
FILE: youtube-8m-wangheda/training_scripts/run-chaining-video.sh
================================================
CUDA_VISIBLE_DEVICES="0" python train.py \
--train_dir="../model/video_chain_moe16_verydeep_combine/" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--model=DeepCombineChainModel \
--label_loss=MultiTaskCrossEntropyLoss \
--train_data_pattern="/Youtube-8M/data/video/train/train*" \
--batch_size=512 \
--multitask=True \
--support_type="label,label,label,label,label,label,label,label" \
--support_loss_percent=0.025 \
--moe_num_mixtures=2 \
--keep_checkpoint_every_n_hours=0.25 \
--num_readers=4 \
--deep_chain_layers=8 \
--deep_chain_relu_cells=128 \
--base_learning_rate=0.004
================================================
FILE: youtube-8m-wangheda/training_scripts/run-cnn-lstm.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/cnnlstmmemory1024_moe8/" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--batch_size=128 \
--moe_num_mixtures=8 \
--model=CnnLstmMemoryModel \
--num_readers=4 \
--base_learning_rate=0.0008 \
--lstm_cells=1024 \
--lstm_layers=2 \
--rnn_swap_memory=True
================================================
FILE: youtube-8m-wangheda/training_scripts/run-cnn-model.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/cnn_model/" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=CnnModel \
--cnn_num_filters=512 \
--moe_num_mixtures=4 \
--num_readers=4 \
--batch_size=128 \
--keep_checkpoint_every_n_hours=0.5 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-wangheda/training_scripts/run-lstm-memory-cell1024.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/lstmmemory_cell1024_layer2_moe8/" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--batch_size=128 \
--moe_num_mixtures=8 \
--model=LstmMemoryModel \
--num_readers=4 \
--base_learning_rate=0.0008 \
--lstm_cells=1024 \
--lstm_layers=2 \
--rnn_swap_memory=True
================================================
FILE: youtube-8m-wangheda/training_scripts/run-lstm-memory-cell2048.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/lstmmemory2048_moe4_batch128/" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--batch_size=128 \
--moe_num_mixtures=4 \
--model=LstmMemoryModel \
--num_readers=4 \
--base_learning_rate=0.0004 \
--keep_checkpoint_every_n_hours=3.0 \
--lstm_cells=2048 \
--lstm_layers=2 \
--rnn_swap_memory=True
================================================
FILE: youtube-8m-wangheda/training_scripts/run-multiple-attention-pooling-positional-embedding.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../x27_model/lstm_positional_attention8max" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--reweight=True \
--sample_vocab_file="resources/train.video_id.vocab" \
--sample_freq_file="resources/train.video_id.freq" \
--model=LstmPositionalAttentionMaxPoolingModel \
--moe_num_mixtures=8 \
--lstm_attentions=8 \
--positional_embedding_size=32 \
--rnn_swap_memory=True \
--base_learning_rate=0.001 \
--num_readers=4 \
--num_epochs=6 \
--batch_size=128 \
--keep_checkpoint_every_n_hour=0.5
================================================
FILE: youtube-8m-wangheda/training_scripts/run-multiscale-cnn-lstm-model.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/multiscale_cnn_lstm_model" \
--train_data_pattern='/Youtube-8M/data/frame/train/*.tfrecord' \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model='MultiscaleCnnLstmModel' \
--multiscale_cnn_lstm_layers=4 \
--moe_num_mixtures=4 \
--multitask=True \
--label_loss=MultiTaskCrossEntropyLoss \
--support_loss_percent=1.0 \
--support_type="label,label,label,label" \
--is_training=True \
--batch_size=128 \
--num_readers=4 \
--num_epochs=5 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-wangheda/training_scripts/run-parallel-lstm-memory.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/lstmparallelmemory1024_moe8" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--batch_size=128 \
--lstm_cells="1024,128" \
--moe_num_mixtures=8 \
--model=LstmParallelMemoryModel \
--rnn_swap_memory=True \
--num_readers=4 \
--num_epochs=5 \
--base_learning_rate=0.0008
================================================
FILE: youtube-8m-wangheda/training_scripts/run-parallel-lstm-output.sh
================================================
CUDA_VISIBLE_DEVICES=1 python train.py \
--train_dir="../model/lstmparallelfinaloutput1024_moe8" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--batch_size=128 \
--lstm_cells="1024,128" \
--moe_num_mixtures=8 \
--model=LstmParallelFinaloutputModel \
--rnn_swap_memory=True \
--num_readers=1 \
--num_epochs=3 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-wangheda/training_scripts/run-temporal-pooling-lstm.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_dir="../model/framehop_lstm/" \
--train_data_pattern="/Youtube-8M/data/frame/train/train*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=FramehopLstmMemoryModel \
--deep_chain_layers=4 \
--deep_chain_relu_cells=256 \
--moe_num_mixtures=4 \
--keep_checkpoint_every_n_hours=1.0 \
--base_learning_rate=0.0008 \
--feature_transformer=IdenticalTransformer \
--num_readers=4 \
--batch_size=128 \
--num_epochs=5 \
--rnn_swap_memory=True \
--lstm_layers=2 \
--lstm_cells="512,64"
================================================
FILE: youtube-8m-wangheda/training_utils/del.py
================================================
import os
check = {}
check_list = []
for filename in os.listdir("."):
if filename.endswith("meta"):
checkpoint = int(filename.split("-")[1].split(".")[0])
check_list.append(checkpoint)
check_list.sort()
for checkpoint in check_list:
if check.has_key(checkpoint / 4000):
print checkpoint
else:
check[checkpoint / 4000] = True
================================================
FILE: youtube-8m-wangheda/training_utils/human_readable_error_analysis.py
================================================
import random
from datetime import datetime
import tensorflow as tf
from tensorflow import flags
FLAGS = flags.FLAGS
if __name__=="__main__":
flags.DEFINE_string("input_error_analysis_file", "", "The file in which every line is a video_id.")
flags.DEFINE_string("output_error_analysis_file", "", "The file in which every line is a video_id.")
flags.DEFINE_string("label_names_file", "", "The file containing the names of labels.")
flags.DEFINE_integer("top_k", 20, "How many predictions are in each line.")
def csv_line(str_list):
return ",".join(map(lambda s: "\""+s+"\"", str_list)) + "\n"
if __name__=="__main__":
top_k = FLAGS.top_k
label_names = dict([line.strip().split(",") for line in open(FLAGS.label_names_file)])
with open(FLAGS.input_error_analysis_file) as fi:
# get rid of the first line
fi.next()
with open(FLAGS.output_error_analysis_file, "w") as fo:
fo.write(csv_line(["Cause", "URL", "ERROR", "field", "list"]))
words_list = []
for line in fi:
words = line.strip().split("\t")
words_list.append(words)
words_list.sort(key=lambda x: -float(x[1]))
for words in words_list:
empty = ""
cause = ""
url = words[0]
error = words[1]
predictions = words[2:2+2*top_k]
p_indices = predictions[0::2]
p_words = map(lambda w: label_names.get(w, "OOV"), p_indices)
p_values = predictions[1::2]
labels = words[2+2*top_k:]
l_indices = labels[0::2]
l_words = map(lambda w: label_names.get(w, "OOV"), l_indices)
l_values = labels[1::2]
fo.write(csv_line([empty, url, error, "predictions"] + p_words))
fo.write(csv_line([empty, empty, empty, empty] + p_values))
fo.write(csv_line([empty, empty, empty, "labels"] + l_words))
fo.write(csv_line([empty, empty, empty, empty] + l_values))
================================================
FILE: youtube-8m-wangheda/training_utils/reweight_sample_freq.py
================================================
import random
import math
from datetime import datetime
import tensorflow as tf
from tensorflow import flags
from collections import defaultdict
FLAGS = flags.FLAGS
if __name__=="__main__":
flags.DEFINE_string("video_id_file", "", "The file in which every line is a video_id.")
flags.DEFINE_string("input_freq_file", "", "The previous weight of each video.")
flags.DEFINE_string("input_error_file", "", "The error of each video.")
flags.DEFINE_string("output_freq_file", "", "Output the corresponding freq of video_ids.")
flags.DEFINE_float("clip_weight", None, "The max value of sample weight. "
"Exceeding part will be randomly distributed to other videos.")
flags.DEFINE_float("discard_weight", None, "The max value of sample weight. "
"The weight exceeding it will be cut to zero with its weight "
"randomly distributed to other videos.")
if __name__=="__main__":
word_list = []
freq_dict = defaultdict(lambda: 1.0)
error_dict = defaultdict(float)
with open(FLAGS.video_id_file) as F:
# read in video_ids
for line in F:
word = line.strip()
if word:
word_list.append(word)
with open(FLAGS.input_freq_file) as F:
# read in frequencies
i = 0
for line in F:
word = word_list[i]
freq = float(line.strip())
freq_dict[word] = freq
i += 1
with open(FLAGS.input_error_file) as F:
# read in errors
for line in F:
words = line.strip().split()
if len(words) == 2:
video_id, error = words
error = float(error)
error_dict[video_id] = error
# compute the new reweight value
epsilon = 1e-6
global_error_rate = sum(error_dict.values()) / len(error_dict)
ratio = math.log((1.0 + epsilon - global_error_rate) / (global_error_rate + epsilon))
for video_id in word_list:
freq_dict[video_id] = freq_dict[video_id] * math.exp(ratio * error_dict[video_id])
pool = 0.0
# discarding weight
if FLAGS.discard_weight:
# discarding
discard = FLAGS.discard_weight
pool = 0.0
for video_id in word_list:
if freq_dict[video_id] > discard:
pool += freq_dict[video_id]
freq_dict[video_id] = 0.0
# clipping weight
if FLAGS.clip_weight:
# clipping
clip = FLAGS.clip_weight
pool = 0.0
for video_id in word_list:
if freq_dict[video_id] > clip:
pool += freq_dict[video_id] - clip
freq_dict[video_id] = clip
if pool > 0:
# re-distributed to other video_id
random.seed(datetime.now())
avg_pool = pool / len(word_list)
for video_id in word_list:
if freq_dict[video_id] > 0:
freq_dict[video_id] += random.random() * 2 * avg_pool
# make the average value 1.0
freq_rel_ratio = max(sum(freq_dict.values()) / len(freq_dict), 1e-6)
for video_id in word_list:
freq_dict[video_id] /= freq_rel_ratio
print "average value in freq_dict =", sum(freq_dict.values()) / len(freq_dict)
# write to output_freq_file
with open(FLAGS.output_freq_file, "w") as F:
lines_to_write = ["%f\n" % freq_dict[video_id] for video_id in word_list]
F.writelines(lines_to_write)
================================================
FILE: youtube-8m-wangheda/training_utils/sample_freq.py
================================================
import random
from datetime import datetime
import tensorflow as tf
from tensorflow import flags
FLAGS = flags.FLAGS
if __name__=="__main__":
flags.DEFINE_string("video_id_file", "", "The file in which every line is a video_id.")
flags.DEFINE_string("output_freq_file", "", "Output the corresponding freq of video_ids.")
if __name__=="__main__":
with open(FLAGS.video_id_file) as F:
word_list = []
freq_dict = {}
# OOV
line = F.next()
for line in F:
word = line.strip()
if word:
word_list.append(word)
freq_dict[word] = 0
# random sample
random.seed(datetime.now())
for i in xrange(len(word_list)):
index = random.randint(0,len(word_list)-1)
word = word_list[index]
freq_dict[word] += 1
# get weight
word_weights = []
word_weights.append(1)
for word in word_list:
word_weights.append(freq_dict[word])
# output weight
with open(FLAGS.output_freq_file, "w") as out:
out.writelines([str(item)+"\n" for item in word_weights])
================================================
FILE: youtube-8m-wangheda/training_utils/select.py
================================================
import os
import sys
if __name__=="__main__":
every = int(sys.argv[1])
check = {}
check_list = []
for filename in os.listdir("."):
if filename.endswith("meta"):
checkpoint = int(filename.split("-")[1].split(".")[0])
check_list.append(checkpoint)
check_list.sort()
for checkpoint in check_list:
if check.has_key(checkpoint / every):
pass
else:
check[checkpoint / every] = True
print checkpoint
================================================
FILE: youtube-8m-wangheda/training_utils/video_original_boosting_error_analysis.py
================================================
import random
from datetime import datetime
import tensorflow as tf
from tensorflow import flags
FLAGS = flags.FLAGS
if __name__=="__main__":
flags.DEFINE_string("train_labels_file", "", "The file in which every line is a video_id and the corresponding labels.")
flags.DEFINE_string("video_id_file", "", "The file in which every line is a video_id.")
flags.DEFINE_string("reweight_freq_file", "", "The file in which every line is a weight used in boosting.")
flags.DEFINE_string("label_names_file", "", "The file containing the names of labels.")
flags.DEFINE_string("output_analysis_file", "", "The file containing the weight, url and labels.")
if __name__=="__main__":
video_ids = [line.strip() for line in open(FLAGS.video_id_file) if len(line.strip()) > 0]
video_freqs = [float(line.strip()) for line in open(FLAGS.reweight_freq_file) if len(line.strip()) > 0]
names_dict = dict([line.strip().split(",") for line in open(FLAGS.label_names_file)])
video_name_dict = {}
label_dict = {}
with open(FLAGS.train_labels_file) as fi:
fi.next()
for line in fi:
if len(line.strip()) > 0:
words = line.strip().split(",")
if len(words) == 2:
video_id, label_ids = words
label_names = " / ".join(map(lambda l: names_dict.get(l, "OOV"), label_ids.split()))
video_name_dict[video_id] = label_names
freq_id_pairs = sorted(zip(video_freqs, video_ids), reverse=True)
with open(FLAGS.output_analysis_file, "w") as fo:
for v_freq, v_id in freq_id_pairs:
print >> fo, v_freq, "\t", "https://www.youtube.com/watch?v=%s"%v_id, "\t", video_name_dict.get(v_id, "OOV")
================================================
FILE: youtube-8m-wangheda/utils.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains a collection of util functions for training and evaluating.
"""
import numpy
import tensorflow as tf
from tensorflow import logging
def Dequantize(feat_vector, max_quantized_value=2, min_quantized_value=-2):
"""Dequantize the feature from the byte format to the float format.
Args:
feat_vector: the input 1-d vector.
max_quantized_value: the maximum of the quantized value.
min_quantized_value: the minimum of the quantized value.
Returns:
A float vector which has the same shape as feat_vector.
"""
assert max_quantized_value > min_quantized_value
quantized_range = max_quantized_value - min_quantized_value
scalar = quantized_range / 255.0
bias = (quantized_range / 512.0) + min_quantized_value
return feat_vector * scalar + bias
def MakeSummary(name, value):
"""Creates a tf.Summary proto with the given name and value."""
summary = tf.Summary()
val = summary.value.add()
val.tag = str(name)
val.simple_value = float(value)
return summary
def AddGlobalStepSummary(summary_writer,
global_step_val,
global_step_info_dict,
summary_scope="Eval"):
"""Add the global_step summary to the Tensorboard.
Args:
summary_writer: Tensorflow summary_writer.
global_step_val: a int value of the global step.
global_step_info_dict: a dictionary of the evaluation metrics calculated for
a mini-batch.
summary_scope: Train or Eval.
Returns:
A string of this global_step summary
"""
this_hit_at_one = global_step_info_dict["hit_at_one"]
this_perr = global_step_info_dict["perr"]
this_loss = global_step_info_dict["loss"]
examples_per_second = global_step_info_dict.get("examples_per_second", -1)
summary_writer.add_summary(
MakeSummary("GlobalStep/" + summary_scope + "_Hit@1", this_hit_at_one),
global_step_val)
summary_writer.add_summary(
MakeSummary("GlobalStep/" + summary_scope + "_Perr", this_perr),
global_step_val)
summary_writer.add_summary(
MakeSummary("GlobalStep/" + summary_scope + "_Loss", this_loss),
global_step_val)
if examples_per_second != -1:
summary_writer.add_summary(
MakeSummary("GlobalStep/" + summary_scope + "_Example_Second",
examples_per_second), global_step_val)
summary_writer.flush()
info = ("global_step {0} | Batch Hit@1: {1:.3f} | Batch PERR: {2:.3f} | Batch Loss: {3:.3f} "
"| Examples_per_sec: {4:.3f}").format(
global_step_val, this_hit_at_one, this_perr, this_loss,
examples_per_second)
return info
def AddEpochSummary(summary_writer,
global_step_val,
epoch_info_dict,
summary_scope="Eval"):
"""Add the epoch summary to the Tensorboard.
Args:
summary_writer: Tensorflow summary_writer.
global_step_val: a int value of the global step.
epoch_info_dict: a dictionary of the evaluation metrics calculated for the
whole epoch.
summary_scope: Train or Eval.
Returns:
A string of this global_step summary
"""
epoch_id = epoch_info_dict["epoch_id"]
avg_hit_at_one = epoch_info_dict["avg_hit_at_one"]
avg_perr = epoch_info_dict["avg_perr"]
avg_loss = epoch_info_dict["avg_loss"]
aps = epoch_info_dict["aps"]
gap = epoch_info_dict["gap"]
mean_ap = numpy.mean(aps)
summary_writer.add_summary(
MakeSummary("Epoch/" + summary_scope + "_Avg_Hit@1", avg_hit_at_one),
global_step_val)
summary_writer.add_summary(
MakeSummary("Epoch/" + summary_scope + "_Avg_Perr", avg_perr),
global_step_val)
summary_writer.add_summary(
MakeSummary("Epoch/" + summary_scope + "_Avg_Loss", avg_loss),
global_step_val)
summary_writer.add_summary(
MakeSummary("Epoch/" + summary_scope + "_MAP", mean_ap),
global_step_val)
summary_writer.add_summary(
MakeSummary("Epoch/" + summary_scope + "_GAP", gap),
global_step_val)
summary_writer.flush()
info = ("epoch/eval number {0} | Avg_Hit@1: {1:.3f} | Avg_PERR: {2:.3f} "
"| MAP: {3:.3f} | GAP: {4:.3f} | Avg_Loss: {5:3f}").format(
epoch_id, avg_hit_at_one, avg_perr, mean_ap, gap, avg_loss)
return info
def GetListOfFeatureNamesAndSizes(feature_names, feature_sizes):
"""Extract the list of feature names and the dimensionality of each feature
from string of comma separated values.
Args:
feature_names: string containing comma separated list of feature names
feature_sizes: string containing comma separated list of feature sizes
Returns:
List of the feature names and list of the dimensionality of each feature.
Elements in the first/second list are strings/integers.
"""
list_of_feature_names = [
feature_names.strip() for feature_names in feature_names.split(',')]
list_of_feature_sizes = [
int(feature_sizes) for feature_sizes in feature_sizes.split(',')]
if len(list_of_feature_names) != len(list_of_feature_sizes):
logging.error("length of the feature names (=" +
str(len(list_of_feature_names)) + ") != length of feature "
"sizes (=" + str(len(list_of_feature_sizes)) + ")")
return list_of_feature_names, list_of_feature_sizes
def clip_gradient_norms(gradients_to_variables, max_norm):
clipped_grads_and_vars = []
for grad, var in gradients_to_variables:
if grad is not None:
if isinstance(grad, tf.IndexedSlices):
tmp = tf.clip_by_norm(grad.values, max_norm)
grad = tf.IndexedSlices(tmp, grad.indices, grad.dense_shape)
else:
grad = tf.clip_by_norm(grad, max_norm)
clipped_grads_and_vars.append((grad, var))
return clipped_grads_and_vars
================================================
FILE: youtube-8m-wangheda/video_level_models.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains model definitions."""
from tensorflow import flags
FLAGS = flags.FLAGS
flags.DEFINE_integer(
"moe_num_mixtures", 2,
"The number of mixtures (excluding the dummy 'expert') used for MoeModel.")
flags.DEFINE_integer(
"deep_chain_layers", 3,
"The number of layers used for DeepChainModel")
flags.DEFINE_integer(
"deep_chain_relu_cells", 200,
"The number of relu cells used for DeepChainModel")
flags.DEFINE_string(
"deep_chain_relu_type", "relu",
"The type of relu cells used for DeepChainModel (options are elu and relu)")
flags.DEFINE_bool(
"deep_chain_use_length", False,
"The number of relu cells used for DeepChainModel")
flags.DEFINE_integer(
"hidden_chain_layers", 4,
"The number of layers used for HiddenChainModel")
flags.DEFINE_integer(
"hidden_chain_relu_cells", 256,
"The number of relu cells used for HiddenChainModel")
flags.DEFINE_integer(
"divergence_model_count", 8,
"The number of models used in divergence enhancement models")
import sys
from os.path import dirname
if dirname(__file__) not in sys.path:
sys.path.append(dirname(__file__))
from all_video_models import *
================================================
FILE: youtube-8m-zhangteng/CONTRIBUTING.md
================================================
# How to contribute
We are accepting patches and contributions to this project. To set expectations,
this project is primarily intended to be a flexible starting point for
researchers working with the YouTube-8M dataset. As such, we would like to keep
it simple. We are more likely to accept small bug fixes and optimizations, and
less likely to accept patches which add significant complexity. For the latter
type of contribution, we recommend creating a Github fork of the project
instead.
If you would like to contribute, there are a few small guidelines you need to
follow.
## Contributor License Agreement
Contributions to any Google project must be accompanied by a Contributor License
Agreement. This is necessary because you own the copyright to your changes, even
after your contribution becomes part of this project. So this agreement simply
gives us permission to use and redistribute your contributions as part of the
project. Head over to to see your current
agreements on file or to sign a new one.
You generally only need to submit a CLA once, so if you've already submitted one
(even if it was for a different project), you probably don't need to do it
again.
## Code reviews
All submissions, including submissions by project members, require review. We
use GitHub pull requests for this purpose. Consult [GitHub Help] for more
information on using pull requests.
[GitHub Help]: https://help.github.com/articles/about-pull-requests/
================================================
FILE: youtube-8m-zhangteng/LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: youtube-8m-zhangteng/README.md
================================================
# YouTube-8M Tensorflow Starter Code
This repo contains starter code for training and evaluating machine learning
models over the [YouTube-8M](https://research.google.com/youtube8m/) dataset.
The code gives an end-to-end working example for reading the dataset, training a
TensorFlow model, and evaluating the performance of the model. Out of the box,
you can train several [model architectures](#overview-of-models) over either
frame-level or video-level features. The code can easily be extended to train
your own custom-defined models.
It is possible to train and evaluate on YouTube-8M in two ways: on Google Cloud
or on your own machine. This README provides instructions for both.
## Table of Contents
* [Running on Google's Cloud Machine Learning Platform](#running-on-googles-cloud-machine-learning-platform)
* [Requirements](#requirements)
* [Testing Locally](#testing-locally)
* [Training on the Cloud over Video-Level Features](#training-on-video-level-features)
* [Evaluation and Inference](#evaluation-and-inference)
* [Accessing Files on Google Cloud](#accessing-files-on-google-cloud)
* [Using Frame-Level Features](#using-frame-level-features)
* [Using Audio Features](#using-audio-features)
* [Running on Your Own Machine](#running-on-your-own-machine)
* [Requirements](#requirements-1)
* [Training on Video-Level Features](#training-on-video-level-features-1)
* [Evaluation and Inference](#evaluation-and-inference-1)
* [Using Frame-Level Features](#using-frame-level-features-1)
* [Using Audio Features](#using-audio-features-1)
* [Ground-Truth Label Files](#ground-truth-label-files)
* [Overview of Models](#overview-of-models)
* [Video-Level Models](#video-level-models)
* [Frame-Level Models](#frame-level-models)
* [Overview of Files](#overview-of-files)
* [Training](#training)
* [Evaluation](#evaluation)
* [Inference](#inference)
* [Misc](#misc)
* [About This Project](#about-this-project)
## Running on Google's Cloud Machine Learning Platform
### Requirements
This option requires you to have an appropriately configured Google Cloud
Platform account. To create and configure your account, please make sure you
follow the instructions [here](https://cloud.google.com/ml/docs/how-tos/getting-set-up).
If you are participating in the Google Cloud & YouTube-8M Video Understanding
Challenge hosted on [kaggle](https://www.kaggle.com/c/youtube8m), see [these instructions](https://www.kaggle.com/c/youtube8m#getting-started-with-google-cloud) instead.
Please also verify that you have Python 2.7+ and Tensorflow 1.0.0 or higher
installed by running the following commands:
```sh
python --version
python -c 'import tensorflow as tf; print(tf.__version__)'
```
### Testing Locally
All gcloud commands should be done from the directory *immediately above* the
source code. You should be able to see the source code directory if you
run 'ls'.
As you are developing your own models, you will want to test them
quickly to flush out simple problems without having to submit them to the cloud.
You can use the `gcloud beta ml local` set of commands for that.
Here is an example command line for video-level training:
```sh
gcloud beta ml local train \
--package-path=youtube-8m --module-name=youtube-8m.train -- \
--train_data_pattern='gs://youtube8m-ml/1/video_level/train/train*.tfrecord' \
--train_dir=/tmp/yt8m_train --start_new_model
```
You might want to download some training shards locally to speed things up and
allow you to work offline. The command below will copy 10 out of the 4096
training data files to the current directory.
```sh
# Downloads 55MB of data.
gsutil cp gs://us.data.yt8m.org/1/video_level/train/traina[0-9].tfrecord .
```
Once you download the files, you can point the job to them using the
'train_data_pattern' argument (i.e. instead of pointing to the "gs://..."
files, you point to the local files).
Once your model is working locally, you can scale up on the Cloud
which is described below.
### Training on the Cloud over Video-Level Features
The following commands will train a model on Google Cloud
over video-level features.
```sh
BUCKET_NAME=gs://${USER}_yt8m_train_bucket
# (One Time) Create a storage bucket to store training logs and checkpoints.
gsutil mb -l us-east1 $BUCKET_NAME
# Submit the training job.
JOB_NAME=yt8m_train_$(date +%Y%m%d_%H%M%S); gcloud --verbosity=debug beta ml jobs \
submit training $JOB_NAME \
--package-path=youtube-8m --module-name=youtube-8m.train \
--staging-bucket=$BUCKET_NAME --region=us-east1 \
--config=youtube-8m/cloudml-gpu.yaml \
-- --train_data_pattern='gs://youtube8m-ml-us-east1/1/video_level/train/train*.tfrecord' \
--train_dir=$BUCKET_NAME/yt8m_train_video_level_logistic_model
```
In the 'gsutil' command above, the 'package-path' flag refers to the directory
containing the 'train.py' script and more generally the python package which
should be deployed to the cloud worker. The module-name refers to the specific
python script which should be executed (in this case the train module).
It may take several minutes before the job starts running on Google Cloud.
When it starts you will see outputs like the following:
```
training step 270| Hit@1: 0.68 PERR: 0.52 Loss: 638.453
training step 271| Hit@1: 0.66 PERR: 0.49 Loss: 635.537
training step 272| Hit@1: 0.70 PERR: 0.52 Loss: 637.564
```
At this point you can disconnect your console by pressing "ctrl-c". The
model will continue to train indefinitely in the Cloud. Later, you can check
on its progress or halt the job by visiting the
[Google Cloud ML Jobs console](https://console.cloud.google.com/ml/jobs).
You can train many jobs at once and use tensorboard to compare their performance
visually.
```sh
tensorboard --logdir=$BUCKET_NAME --port=8080
```
Once tensorboard is running, you can access it at the following url:
[http://localhost:8080](http://localhost:8080).
If you are using Google Cloud Shell, you can instead click the Web Preview button
on the upper left corner of the Cloud Shell window and select "Preview on port 8080".
This will bring up a new browser tab with the Tensorboard view.
### Evaluation and Inference
Here's how to evaluate a model on the validation dataset:
```sh
JOB_TO_EVAL=yt8m_train_video_level_logistic_model
JOB_NAME=yt8m_eval_$(date +%Y%m%d_%H%M%S); gcloud --verbosity=debug beta ml jobs \
submit training $JOB_NAME \
--package-path=youtube-8m --module-name=youtube-8m.eval \
--staging-bucket=$BUCKET_NAME --region=us-east1 \
--config=youtube-8m/cloudml-gpu.yaml \
-- --eval_data_pattern='gs://youtube8m-ml-us-east1/1/video_level/validate/validate*.tfrecord' \
--train_dir=$BUCKET_NAME/${JOB_TO_EVAL} --run_once=True
```
And here's how to perform inference with a model on the test set:
```sh
JOB_TO_EVAL=yt8m_train_video_level_logistic_model
JOB_NAME=yt8m_inference_$(date +%Y%m%d_%H%M%S); gcloud --verbosity=debug beta ml jobs \
submit training $JOB_NAME \
--package-path=youtube-8m --module-name=youtube-8m.inference \
--staging-bucket=$BUCKET_NAME --region=us-east1 \
--config=youtube-8m/cloudml-gpu.yaml \
-- --input_data_pattern='gs://youtube8m-ml/1/video_level/test/test*.tfrecord' \
--train_dir=$BUCKET_NAME/${JOB_TO_EVAL} \
--output_file=$BUCKET_NAME/${JOB_TO_EVAL}/predictions.csv
```
Note the confusing use of 'training' in the above gcloud commands. Despite the
name, the 'training' argument really just offers a cloud hosted
python/tensorflow service. From the point of view of the Cloud Platform, there
is no distinction between our training and inference jobs. The Cloud ML platform
also offers specialized functionality for prediction with
Tensorflow models, but discussing that is beyond the scope of this readme.
Once these job starts executing you will see outputs similar to the
following for the evaluation code:
```
examples_processed: 1024 | global_step 447044 | Batch Hit@1: 0.782 | Batch PERR: 0.637 | Batch Loss: 7.821 | Examples_per_sec: 834.658
```
and the following for the inference code:
```
num examples processed: 8192 elapsed seconds: 14.85
```
### Accessing Files on Google Cloud
You can browse the storage buckets you created on Google Cloud, for example, to
access the trained models, prediction CSV files, etc. by visiting the
[Google Cloud storage browser](https://console.cloud.google.com/storage/browser).
Alternatively, you can use the 'gsutil' command to download the files directly.
For example, to download the output of the inference code from the previous
section to your local machine, run:
```
gsutil cp $BUCKET_NAME/${JOB_TO_EVAL}/predictions.csv .
```
### Using Frame-Level Features
Append
```sh
--frame_features=True --model=FrameLevelLogisticModel --feature_names="rgb" \
--feature_sizes="1024" --batch_size=128 \
--train_dir=$BUCKET_NAME/yt8m_train_frame_level_logistic_model
```
to the 'gcloud' commands given above, and change 'video_level' in paths to
'frame_level'. Here is a sample command to kick-off a frame-level job:
```sh
JOB_NAME=yt8m_train_$(date +%Y%m%d_%H%M%S); gcloud --verbosity=debug beta ml jobs \
submit training $JOB_NAME \
--package-path=youtube-8m --module-name=youtube-8m.train \
--staging-bucket=$BUCKET_NAME --region=us-east1 \
--config=youtube-8m/cloudml-gpu.yaml \
-- --train_data_pattern='gs://youtube8m-ml-us-east1/1/frame_level/train/train*.tfrecord' \
--frame_features=True --model=FrameLevelLogisticModel --feature_names="rgb" \
--feature_sizes="1024" --batch_size=128 \
--train_dir=$BUCKET_NAME/yt8m_train_frame_level_logistic_model
```
The 'FrameLevelLogisticModel' is designed to provide equivalent results to a
logistic model trained over the video-level features. Please look at the
'video_level_models.py' or 'frame_level_models.py' files to see how to implement
your own models.
### Using Audio Features
The feature files (both Frame-Level and Video-Level) contain two sets of
features: 1) visual and 2) audio. The code defaults to using the visual
features only, but it is possible to use audio features instead of (or besides)
visual features. To specify the (combination of) features to use you must set
`--feature_names` and `--feature_sizes` flags. The visual and audio features are
called 'rgb' and 'audio' and have 1024 and 128 dimensions, respectively.
The two flags take a comma-separated list of values in string. For example, to
use audio-visual Video-Level features the flags must be set as follows:
```
--feature_names="mean_rgb, mean_audio" --feature_sizes="1024, 128"
```
Similarly, to use audio-visual Frame-Level features use:
```
--feature_names="rgb, audio" --feature_sizes="1024, 128"
```
**NOTE:** Make sure the set of features and the order in which the appear in the
lists provided to the two flags above match. Also, the order must match when
running training, evaluation, or inference.
## Running on Your Own Machine
### Requirements
The starter code requires Tensorflow. If you haven't installed it yet, follow
the instructions on [tensorflow.org](https://www.tensorflow.org/install/).
This code has been tested with Tensorflow 1.0.0. Going forward, we will continue
to target the latest released version of Tensorflow.
Please verify that you have Python 2.7+ and Tensorflow 1.0.0 or higher
installed by running the following commands:
```sh
python --version
python -c 'import tensorflow as tf; print(tf.__version__)'
```
You can find complete instructions for downloading the dataset on the
[YouTube-8M website](https://research.google.com/youtube8m/download.html).
We recommend downloading the smaller video-level features dataset first when
getting started. To do that, run:
```
mkdir -p features; cd features
curl data.yt8m.org/download.py | partition=1/video_level/train mirror=us python
```
This will download the full set of video level features, which takes up 31GB
of space.
If you are located outside of North America, you should change the flag 'mirror'
to 'eu' for Europe or 'asia' for Asia to speed up the transfer of the files.
Change 'train' to 'validate'/'test' and re-run the command to download the
other splits of the dataset.
Change 'video_level' to 'frame_level' to download the frame-level features. The
complete frame-level features take about 1.71TB of space. You can set the
environment variable 'shard' to 'm,n' to download only m/n-th of the data. For
example, to download 1/100-th of the frame-level features from the training set,
run:
```
curl data.yt8m.org/download.py | shard=1,100 partition=1/frame_level/train mirror=us python
```
### Training on Video-Level Features
To start training a logistic model on the video-level features, run
```sh
MODEL_DIR=/tmp/yt8m
python train.py --train_data_pattern='/path/to/features/train*.tfrecord' --train_dir=$MODEL_DIR/video_level_logistic_model
```
Since the dataset is sharded into 4096 individual files, we use a wildcard (\*)
to represent all of those files.
By default, the training code will frequently write _checkpoint_ files (i.e.
values of all trainable parameters, at the current training iteration). These
will be written to the `--train_dir`. If you re-use a `--train_dir`, the trainer
will first restore the latest checkpoint written in that directory. This only
works if the architecture of the checkpoint matches the graph created by the
training code. If you are in active development/debugging phase, consider
adding `--start_new_model` flag to your run configuration.
### Evaluation and Inference
To evaluate the model, run
```sh
python eval.py --eval_data_pattern='/path/to/features/validate*.tfrecord' --train_dir=$MODEL_DIR/video_level_logistic_model --run_once=True
```
As the model is training or evaluating, you can view the results on tensorboard
by running
```sh
tensorboard --logdir=$MODEL_DIR
```
and navigating to http://localhost:6006 in your web browser.
When you are happy with your model, you can generate a csv file of predictions
from it by running
```sh
python inference.py --output_file=$MODEL_DIR/video_level_logistic_model/predictions.csv --input_data_pattern='/path/to/features/test*.tfrecord' --train_dir=$MODEL_DIR/video_level_logistic_model
```
This will output the top 20 predicted labels from the model for every example
to 'predictions.csv'.
### Using Frame-Level Features
Follow the same instructions as above, appending
`--frame_features=True --model=FrameLevelLogisticModel --feature_names="rgb"
--feature_sizes="1024" --train_dir=$MODEL_DIR/frame_level_logistic_model`
for the 'train.py', 'eval.py', and 'inference.py' scripts.
The 'FrameLevelLogisticModel' is designed to provide equivalent results to a
logistic model trained over the video-level features. Please look at the
'models.py' file to see how to implement your own models.
### Using Audio Features
See [Using Audio Features](#using-audio-features) section above.
### Ground-Truth Label Files
We also provide CSV files containing the ground-truth label information of the
'train' and 'validation' partitions of the dataset. These files can be
downloaded using 'gsutil' command:
```
gsutil cp gs://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv /destination/folder/
gsutil cp gs://us.data.yt8m.org/1/ground_truth_labels/validate_labels.csv /destination/folder/
```
or directly using the following links:
* [http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv](http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv)
* [http://us.data.yt8m.org/1/ground_truth_labels/validate_labels.csv](http://us.data.yt8m.org/1/ground_truth_labels/validate_labels.csv)
Each line in the files starts with the video id and is followed by the list of
ground-truth labels corresponding to that video. For example, for a video with
id 'VIDEO_ID' and two lables 'LABLE1' and 'LABEL2' we store the following line:
```
VIDEO_ID,LABEL1 LABEL2
```
## Overview of Models
This sample code contains implementations of the models given in the
[YouTube-8M technical report](https://arxiv.org/abs/1609.08675).
### Video-Level Models
* `LogisticModel`: Linear projection of the output features into the label
space, followed by a sigmoid function to convert logit
values to probabilities.
* `MoeModel`: A per-class softmax distribution over a configurable number of
logistic classifiers. One of the classifiers in the mixture
is not trained, and always predicts 0.
### Frame-Level Models
* `LstmModel`: Processes the features for each frame using a multi-layered
LSTM neural net. The final internal state of the LSTM
is input to a video-level model for classification. Note that
you will need to change the learning rate to 0.001 when using
this model.
* `DbofModel`: Projects the features for each frame into a higher dimensional
'clustering' space, pools across frames in that space, and then
uses a video-level model to classify the now aggregated features.
* `FrameLevelLogisticModel`: Equivalent to 'LogisticModel', but performs
average-pooling on the fly over frame-level
features rather than using pre-aggregated features.
## Overview of Files
### Training
* `train.py`: The primary script for training models.
* `losses.py`: Contains definitions for loss functions.
* `models.py`: Contains the base class for defining a model.
* `video_level_models.py`: Contains definitions for models that take
aggregated features as input.
* `frame_level_models.py`: Contains definitions for models that take frame-
level features as input.
* `model_util.py`: Contains functions that are of general utility for
implementing models.
* `readers.py`: Contains definitions for the Video dataset and Frame
dataset readers.
### Evaluation
* `eval.py`: The primary script for evaluating models.
* `eval_util.py`: Provides a class that calculates all evaluation metrics.
* `average_precision_calculator.py`: Functions for calculating
average precision.
* `mean_average_precision_calculator.py`: Functions for calculating mean
average precision.
### Inference
* `inference.py`: Generates an output file containing predictions of
the model over a set of videos.
### Misc
* `README.md`: This documentation.
* `utils.py`: Common functions.
## About This Project
This project is meant help people quickly get started working with the
[YouTube-8M](https://research.google.com/youtube8m/) dataset.
This is not an official Google product.
================================================
FILE: youtube-8m-zhangteng/YM_framemean.py
================================================
from __future__ import print_function
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
import numpy as np
import os, fnmatch
import threading
import Queue
import utils
flags.DEFINE_string("src_path", "/Youtube-8M/data/frame/train", "")
flags.DEFINE_string("des_path", "./mean_std.tfrecord", "")
def get_frame_input_feature(input_file):
features = []
record_iterator = tf.python_io.tf_record_iterator(path=input_file)
for i, string_record in enumerate(record_iterator):
example = tf.train.SequenceExample()
example.ParseFromString(string_record)
# traverse the Example format to get data
video_id = example.context.feature['video_id'].bytes_list.value[0]
label = example.context.feature['labels'].int64_list.value[:]
rgbs = []
audios = []
rgb_feature = example.feature_lists.feature_list['rgb'].feature
for i in range(len(rgb_feature)):
rgb = np.fromstring(rgb_feature[i].bytes_list.value[0], dtype=np.uint8).astype(np.float32)
rgb = utils.Dequantize(rgb, 2, -2)
rgbs.append(rgb)
audio_feature = example.feature_lists.feature_list['audio'].feature
for i in range(len(audio_feature)):
audio = np.fromstring(audio_feature[i].bytes_list.value[0], dtype=np.uint8).astype(np.float32)
audio = utils.Dequantize(audio, 2, -2)
audios.append(audio)
rgbs = np.array(rgbs)
audios = np.array(audios)
features.append((video_id, label, rgbs, audios))
return features
def get_video_input_feature(input_file):
features = []
record_iterator = tf.python_io.tf_record_iterator(path=input_file)
for i, string_record in enumerate(record_iterator):
example = tf.train.Example()
example.ParseFromString(string_record)
# traverse the Example format to get data
video_id = example.features.feature['video_id'].bytes_list.value[0]
label = example.features.feature['labels'].int64_list.value[:]
mean_rgb = example.features.feature['mean_rgb'].float_list.value[:]
mean_audio = example.features.feature['mean_audio'].float_list.value[:]
std_rgb = example.features.feature['std_rgb'].float_list.value[:]
std_audio = example.features.feature['std_audio'].float_list.value[:]
features.append((video_id, label, mean_rgb, mean_audio, std_rgb, std_audio))
return features
def get_output_files(features, feature_names):
feature_maps = {}
for feature_index in range(len(feature_names)):
feature_maps[feature_names[feature_index]] = tf.train.Feature(
float_list=tf.train.FloatList(value=features[feature_index]))
example = tf.train.Example(features=tf.train.Features(feature=feature_maps))
return example
def get_output_feature(video_id, labels, features, feature_names):
feature_maps = {'video_id': tf.train.Feature(bytes_list=tf.train.BytesList(value=[video_id])),
'labels': tf.train.Feature(int64_list=tf.train.Int64List(value=labels))}
for feature_index in range(len(feature_names)):
feature_maps[feature_names[feature_index]] = tf.train.Feature(
float_list=tf.train.FloatList(value=features[feature_index]))
example = tf.train.Example(features=tf.train.Features(feature=feature_maps))
return example
def read_batch_files(q,files):
mean_features = []
std_features = []
mean_all = []
std_all = []
for file in files:
print('processing '+file)
features = get_frame_input_feature(flags.FLAGS.src_path+'/'+file)
features_all = []
for i, feature in enumerate(features):
feature_1 = np.reshape(np.mean(feature[2], axis=0),[feature[2].shape[1],1])
feature_2 = np.reshape(np.mean(feature[3], axis=0),[feature[3].shape[1],1])
feature_3 = np.reshape(np.std(feature[2], axis=0),[feature[2].shape[1],1])
feature_4 = np.reshape(np.std(feature[3], axis=0),[feature[3].shape[1],1])
if features_all:
feature_1 = np.concatenate([features_all[0],feature_1],axis=1)
feature_2 = np.concatenate([features_all[1],feature_2],axis=1)
feature_3 = np.concatenate([features_all[2],feature_3],axis=1)
feature_4 = np.concatenate([features_all[3],feature_4],axis=1)
features_all = [feature_1,feature_2,feature_3,feature_4]
else:
features_all = [feature_1,feature_2,feature_3,feature_4]
mean_1 = []
std_1 = []
mean_all_1 = []
std_all_1 = []
for i in range(4):
mean_1.append(np.reshape(np.mean(features_all[i], axis=1),[features_all[i].shape[0],1]))
std_1.append(np.reshape(np.std(features_all[i], axis=1),[features_all[i].shape[0],1]))
mean_all_1.append(np.reshape(np.mean(features_all[i]),[1,1]))
std_all_1.append(np.reshape(np.std(features_all[i]),[1,1]))
if mean_features:
for i in range(4):
mean_features[i] = np.concatenate([mean_features[i],mean_1[i]],axis=1)
std_features[i] = np.concatenate([std_features[i],std_1[i]],axis=1)
mean_all[i] = np.concatenate([mean_all[i],mean_all_1[i]],axis=1)
std_all[i] = np.concatenate([std_all[i],std_all_1[i]],axis=1)
else:
for i in range(4):
mean_features.append(mean_1[i])
std_features.append(std_1[i])
mean_all.append(mean_all_1[i])
std_all.append(std_all_1[i])
q.put([mean_features,std_features,mean_all,std_all])
class myThread(threading.Thread):
def __init__(self, threadID, name, files, q):
threading.Thread.__init__(self)
self.threadID = threadID
self.name = name
self.files = files
self.q = q
def run(self):
print("Starting " + self.name)
read_batch_files(self.q, self.files)
def main():
files = fnmatch.filter(os.listdir(flags.FLAGS.src_path), '*.tfrecord')
threads =[]
thread_num = 10
fstep = int(len(files)/thread_num)+1
q = Queue.Queue()
for i in range(thread_num):
thread = myThread(i, "Thread-%d" % i, files[fstep*i:fstep*(i+1)], q)
threads.append(thread)
# Wait for all threads to complete
print(threads)
mean_features = []
std_features = []
mean_all = []
std_all = []
#Start all threads in thread pool
for t in threads:
t.start()
for t in threads:
t.join()
while not q.empty():
response = q.get()
mean_features.extend(response[0])
std_features.extend(response[1])
mean_all.extend(response[2])
std_all.extend(response[3])
for i in range(4):
mean_features[i] = np.mean(mean_features[i],axis=1)
std_features[i] = np.mean(std_features[i],axis=1)
mean_all[i] = np.mean(mean_all[i],axis=1)
std_all[i] = np.mean(std_all[i],axis=1)
mean_all = np.array(mean_all)
std_all = np.array(std_all)
mean_features.extend(std_features)
mean_features.append(mean_all)
mean_features.append(std_all)
writer = tf.python_io.TFRecordWriter(flags.FLAGS.des_path)
example = get_output_files(mean_features,['mean_rgb_mean','mean_audio_mean','std_rgb_mean','std_audio_mean',
'mean_rgb_std','mean_audio_std','std_rgb_std','std_audio_std','mean_4','std_4'])
serialized = example.SerializeToString()
writer.write(serialized)
writer.close()
if __name__=='__main__':
main()
================================================
FILE: youtube-8m-zhangteng/YM_labels_matrix.py
================================================
from __future__ import print_function
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
import numpy as np
import utils
flags.DEFINE_string("src_path_1", "/home/share/zhangt/mywork/youtube-8m-master/data/video/train_my/trainL_.tfrecord", "")
flags.DEFINE_string("src_path_2", "/home/share/Youtube-8M/data/video/validate/*.tfrecord", "")
def get_frame_input_feature(input_file):
features = []
record_iterator = tf.python_io.tf_record_iterator(path=input_file)
for i, string_record in enumerate(record_iterator):
example = tf.train.SequenceExample()
example.ParseFromString(string_record)
# traverse the Example format to get data
video_id = example.context.feature['video_id'].bytes_list.value[0]
label = example.context.feature['labels'].int64_list.value[:]
rgbs = []
audios = []
rgb_feature = example.feature_lists.feature_list['rgb'].feature
for i in range(len(rgb_feature)):
rgb = np.fromstring(rgb_feature[i].bytes_list.value[0], dtype=np.uint8).astype(np.float32)
rgb = utils.Dequantize(rgb, 2, -2)
rgbs.append(rgb)
audio_feature = example.feature_lists.feature_list['audio'].feature
for i in range(len(audio_feature)):
audio = np.fromstring(audio_feature[i].bytes_list.value[0], dtype=np.uint8).astype(np.float32)
audio = utils.Dequantize(audio, 2, -2)
audios.append(audio)
rgbs = np.array(rgbs)
audios = np.array(audios)
features.append((video_id, label, rgbs, audios))
return features
def get_video_input_feature(input_file):
features = []
record_iterator = tf.python_io.tf_record_iterator(path=input_file)
for i, string_record in enumerate(record_iterator):
example = tf.train.Example()
example.ParseFromString(string_record)
# traverse the Example format to get data
video_id = example.features.feature['video_id'].bytes_list.value[0]
label = example.features.feature['labels'].int64_list.value[:]
mean_rgb = example.features.feature['mean_rgb'].float_list.value[:]
mean_audio = example.features.feature['mean_audio'].float_list.value[:]
std_rgb = example.features.feature['std_rgb'].float_list.value[:]
std_audio = example.features.feature['std_audio'].float_list.value[:]
#features.append(label)
features.append((video_id, label, mean_rgb, mean_audio, std_rgb, std_audio))
return features
def get_output_feature(video_id, labels, features, feature_names):
feature_maps = {'video_id': tf.train.Feature(bytes_list=tf.train.BytesList(value=[video_id])),
'labels': tf.train.Feature(int64_list=tf.train.Int64List(value=labels))}
for feature_index in range(len(feature_names)):
feature_maps[feature_names[feature_index]] = tf.train.Feature(
float_list=tf.train.FloatList(value=features[feature_index]))
example = tf.train.Example(features=tf.train.Features(feature=feature_maps))
return example
def main():
files = tf.gfile.Glob(flags.FLAGS.src_path_1)
labels_uni = np.zeros([4716,1])
labels_matrix = np.zeros([4716,4716])
for file in files:
labels_all = get_video_input_feature(file)
print(len(labels_all[0][2]),len(labels_all[0][3]),len(labels_all[0][4]),len(labels_all[0][5]))
"""
for labels in labels_all:
for i in range(len(labels)):
labels_uni[labels[i]] += 1
for j in range(len(labels)):
labels_matrix[labels[i],labels[j]] += 1
labels_matrix = labels_matrix/labels_uni
labels_matrix = labels_matrix/(np.sum(labels_matrix,axis=0)-1.0)
for i in range(4716):
labels_matrix[i,i] = 1.0
np.savetxt('labels_uni.out', labels_uni, delimiter=',')
np.savetxt('labels_matrix.out', labels_matrix, delimiter=',')"""
if __name__=='__main__':
main()
================================================
FILE: youtube-8m-zhangteng/YM_labels_model.py
================================================
from __future__ import print_function
import tensorflow as tf
from tensorflow import flags
import numpy as np
import csv
flags.DEFINE_string("src_path_1", "predictions_best.csv", "")
def main():
labels_uni = np.zeros([4716,1])
with open(flags.FLAGS.src_path_1, "rt", encoding='utf-8') as csvfile:
spamreader = csv.reader(csvfile)
line_num = 0
for row in spamreader:
line_num += 1
print('the '+str(line_num)+'th file is processing')
if line_num==1:
continue
lbs = row[1].split()
for i in range(0,len(lbs),2):
labels_uni[int(lbs[i])] += 1
np.savetxt('labels_model.out', labels_uni, delimiter=',')
if __name__=='__main__':
main()
================================================
FILE: youtube-8m-zhangteng/YM_labels_vocab.py
================================================
from __future__ import print_function
from tensorflow import flags
import numpy as np
import random
import csv
flags.DEFINE_string("src_path_1", "vocabulary.csv", "")
def main():
rootclass = {}
with open(flags.FLAGS.src_path_1, "rt", encoding='utf-8') as csvfile:
spamreader = csv.reader(csvfile)
line_num = 0
for row in spamreader:
line_num += 1
print('the '+str(line_num)+'th file is processing')
if line_num==1:
continue
if row[5] in rootclass:
rootclass[row[5]].append(line_num-2)
else:
rootclass[row[5]] = [line_num-2]
labels_ordered = []
for x in rootclass:
labels_ordered.extend(rootclass[x])
labels_ordered = [int(l) for l in labels_ordered]
reverse_ordered = np.zeros([4716,1])
for i in range(len(labels_ordered)):
reverse_ordered[labels_ordered[i]] = i
print(len(rootclass))
print(labels_ordered)
np.savetxt('labels_ordered.out', reverse_ordered, delimiter=',')
random.shuffle(labels_ordered)
reverse_unordered = np.zeros([4716,1])
for i in range(len(labels_ordered)):
reverse_unordered[labels_ordered[i]] = i
print(labels_ordered)
np.savetxt('labels_unordered.out', reverse_unordered, delimiter=',')
labels_class = np.zeros([len(rootclass),4716])
flag = 0
for x in rootclass:
for i in rootclass[x]:
labels_class[flag,i] = 1
flag +=1
np.savetxt('labels_class.out', labels_class)
if __name__=='__main__':
main()
================================================
FILE: youtube-8m-zhangteng/YM_readframefeature.py
================================================
from __future__ import print_function
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
import numpy as np
import utils
flags.DEFINE_string("src_path", "./train-A.tfrecord", "")
flags.DEFINE_string("des_path", "./train-A-video.tfrecord", "")
def get_frame_input_feature(input_file):
features = []
record_iterator = tf.python_io.tf_record_iterator(path=input_file)
for i, string_record in enumerate(record_iterator):
example = tf.train.SequenceExample()
example.ParseFromString(string_record)
# traverse the Example format to get data
video_id = example.context.feature['video_id'].bytes_list.value[0]
label = example.context.feature['labels'].int64_list.value[:]
rgbs = []
audios = []
rgb_feature = example.feature_lists.feature_list['rgb'].feature
for i in range(len(rgb_feature)):
rgb = np.fromstring(rgb_feature[i].bytes_list.value[0], dtype=np.uint8).astype(np.float32)
rgb = utils.Dequantize(rgb, 2, -2)
rgbs.append(rgb)
audio_feature = example.feature_lists.feature_list['audio'].feature
for i in range(len(audio_feature)):
audio = np.fromstring(audio_feature[i].bytes_list.value[0], dtype=np.uint8).astype(np.float32)
audio = utils.Dequantize(audio, 2, -2)
audios.append(audio)
rgbs = np.array(rgbs)
audios = np.array(audios)
features.append((video_id, label, rgbs, audios))
return features
def get_video_input_feature(input_file):
features = []
record_iterator = tf.python_io.tf_record_iterator(path=input_file)
for i, string_record in enumerate(record_iterator):
example = tf.train.Example()
example.ParseFromString(string_record)
# traverse the Example format to get data
video_id = example.features.feature['video_id'].bytes_list.value[0]
label = example.features.feature['labels'].int64_list.value[:]
mean_rgb = example.features.feature['mean_rgb'].float_list.value[:]
mean_audio = example.features.feature['mean_audio'].float_list.value[:]
std_rgb = example.features.feature['std_rgb'].float_list.value[:]
std_audio = example.features.feature['std_audio'].float_list.value[:]
features.append((video_id, label, mean_rgb, mean_audio, std_rgb, std_audio))
return features
def get_output_feature(video_id, labels, features, feature_names):
feature_maps = {'video_id': tf.train.Feature(bytes_list=tf.train.BytesList(value=[video_id])),
'labels': tf.train.Feature(int64_list=tf.train.Int64List(value=labels))}
for feature_index in range(len(feature_names)):
feature_maps[feature_names[feature_index]] = tf.train.Feature(
float_list=tf.train.FloatList(value=features[feature_index]))
example = tf.train.Example(features=tf.train.Features(feature=feature_maps))
return example
def main():
features = get_frame_input_feature(flags.FLAGS.src_path)
writer = tf.python_io.TFRecordWriter(flags.FLAGS.des_path)
#mean_features = np.loadtxt('frame_means.out', delimiter=',')
#std_features = np.loadtxt('frame_stds.out', delimiter=',')
for i, feature in enumerate(features):
"""
feature_1 = (np.mean(feature[2], axis=0)-mean_features[0])/std_features[0]
feature_2 = (np.mean(feature[3], axis=0)-mean_features[1])/std_features[1]
feature_3 = (np.std(feature[2], axis=0)-mean_features[2])/std_features[2]
feature_4 = (np.std(feature[3], axis=0)-mean_features[3])/std_features[3]"""
feature_1 = np.mean(feature[2], axis=0)
feature_2 = np.mean(feature[3], axis=0)
feature_3 = np.std(feature[2], axis=0)
feature_4 = np.std(feature[3], axis=0)
example = get_output_feature(feature[0], feature[1], [feature_1, feature_2, feature_3, feature_4],
['mean_rgb','mean_audio','std_rgb','std_audio'])
serialized = example.SerializeToString()
writer.write(serialized)
writer.close()
"""
features = get_video_input_feature(flags.FLAGS.des_path)
features_all = []
for i, feature in enumerate(features):
feature_1 = np.reshape(np.array(feature[2]),[len(feature[2]),1])
feature_2 = np.reshape(np.array(feature[3]),[len(feature[3]),1])
feature_3 = np.reshape(np.array(feature[4]),[len(feature[4]),1])
feature_4 = np.reshape(np.array(feature[5]),[len(feature[5]),1])
if features_all:
feature_1 = np.concatenate([features_all[0],feature_1],axis=1)
feature_2 = np.concatenate([features_all[1],feature_2],axis=1)
feature_3 = np.concatenate([features_all[2],feature_3],axis=1)
feature_4 = np.concatenate([features_all[3],feature_4],axis=1)
features_all = [feature_1,feature_2,feature_3,feature_4]
else:
features_all = [feature_1,feature_2,feature_3,feature_4]
mean_features = []
std_features = []
for i in range(len(features_all)):
mean_features.append(np.mean(features_all[i],axis=1))
std_features.append(np.std(features_all[i],axis=1))
print(mean_features,std_features)"""
if __name__=='__main__':
main()
================================================
FILE: youtube-8m-zhangteng/__init__.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
================================================
FILE: youtube-8m-zhangteng/average_precision_calculator.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Calculate or keep track of the interpolated average precision.
It provides an interface for calculating interpolated average precision for an
entire list or the top-n ranked items. For the definition of the
(non-)interpolated average precision:
http://trec.nist.gov/pubs/trec15/appendices/CE.MEASURES06.pdf
Example usages:
1) Use it as a static function call to directly calculate average precision for
a short ranked list in the memory.
```
import random
p = np.array([random.random() for _ in xrange(10)])
a = np.array([random.choice([0, 1]) for _ in xrange(10)])
ap = average_precision_calculator.AveragePrecisionCalculator.ap(p, a)
```
2) Use it as an object for long ranked list that cannot be stored in memory or
the case where partial predictions can be observed at a time (Tensorflow
predictions). In this case, we first call the function accumulate many times
to process parts of the ranked list. After processing all the parts, we call
peek_interpolated_ap_at_n.
```
p1 = np.array([random.random() for _ in xrange(5)])
a1 = np.array([random.choice([0, 1]) for _ in xrange(5)])
p2 = np.array([random.random() for _ in xrange(5)])
a2 = np.array([random.choice([0, 1]) for _ in xrange(5)])
# interpolated average precision at 10 using 1000 break points
calculator = average_precision_calculator.AveragePrecisionCalculator(10)
calculator.accumulate(p1, a1)
calculator.accumulate(p2, a2)
ap3 = calculator.peek_ap_at_n()
```
"""
import heapq
import random
import numbers
import numpy
class AveragePrecisionCalculator(object):
"""Calculate the average precision and average precision at n."""
def __init__(self, top_n=None):
"""Construct an AveragePrecisionCalculator to calculate average precision.
This class is used to calculate the average precision for a single label.
Args:
top_n: A positive Integer specifying the average precision at n, or
None to use all provided data points.
Raises:
ValueError: An error occurred when the top_n is not a positive integer.
"""
if not ((isinstance(top_n, int) and top_n >= 0) or top_n is None):
raise ValueError("top_n must be a positive integer or None.")
self._top_n = top_n # average precision at n
self._total_positives = 0 # total number of positives have seen
self._heap = [] # max heap of (prediction, actual)
@property
def heap_size(self):
"""Gets the heap size maintained in the class."""
return len(self._heap)
@property
def num_accumulated_positives(self):
"""Gets the number of positive samples that have been accumulated."""
return self._total_positives
def accumulate(self, predictions, actuals, num_positives=None):
"""Accumulate the predictions and their ground truth labels.
After the function call, we may call peek_ap_at_n to actually calculate
the average precision.
Note predictions and actuals must have the same shape.
Args:
predictions: a list storing the prediction scores.
actuals: a list storing the ground truth labels. Any value
larger than 0 will be treated as positives, otherwise as negatives.
num_positives = If the 'predictions' and 'actuals' inputs aren't complete,
then it's possible some true positives were missed in them. In that case,
you can provide 'num_positives' in order to accurately track recall.
Raises:
ValueError: An error occurred when the format of the input is not the
numpy 1-D array or the shape of predictions and actuals does not match.
"""
if len(predictions) != len(actuals):
raise ValueError("the shape of predictions and actuals does not match.")
if not num_positives is None:
if not isinstance(num_positives, numbers.Number) or num_positives < 0:
raise ValueError("'num_positives' was provided but it wan't a nonzero number.")
if not num_positives is None:
self._total_positives += num_positives
else:
self._total_positives += numpy.size(numpy.where(actuals > 0))
topk = self._top_n
heap = self._heap
for i in range(numpy.size(predictions)):
if topk is None or len(heap) < topk:
heapq.heappush(heap, (predictions[i], actuals[i]))
else:
if predictions[i] > heap[0][0]: # heap[0] is the smallest
heapq.heappop(heap)
heapq.heappush(heap, (predictions[i], actuals[i]))
def clear(self):
"""Clear the accumulated predictions."""
self._heap = []
self._total_positives = 0
def peek_ap_at_n(self):
"""Peek the non-interpolated average precision at n.
Returns:
The non-interpolated average precision at n (default 0).
If n is larger than the length of the ranked list,
the average precision will be returned.
"""
if self.heap_size <= 0:
return 0
predlists = numpy.array(list(zip(*self._heap)))
ap = self.ap_at_n(predlists[0],
predlists[1],
n=self._top_n,
total_num_positives=self._total_positives)
return ap
@staticmethod
def ap(predictions, actuals):
"""Calculate the non-interpolated average precision.
Args:
predictions: a numpy 1-D array storing the sparse prediction scores.
actuals: a numpy 1-D array storing the ground truth labels. Any value
larger than 0 will be treated as positives, otherwise as negatives.
Returns:
The non-interpolated average precision at n.
If n is larger than the length of the ranked list,
the average precision will be returned.
Raises:
ValueError: An error occurred when the format of the input is not the
numpy 1-D array or the shape of predictions and actuals does not match.
"""
return AveragePrecisionCalculator.ap_at_n(predictions,
actuals,
n=None)
@staticmethod
def ap_at_n(predictions, actuals, n=20, total_num_positives=None):
"""Calculate the non-interpolated average precision.
Args:
predictions: a numpy 1-D array storing the sparse prediction scores.
actuals: a numpy 1-D array storing the ground truth labels. Any value
larger than 0 will be treated as positives, otherwise as negatives.
n: the top n items to be considered in ap@n.
total_num_positives : (optionally) you can specify the number of total
positive
in the list. If specified, it will be used in calculation.
Returns:
The non-interpolated average precision at n.
If n is larger than the length of the ranked list,
the average precision will be returned.
Raises:
ValueError: An error occurred when
1) the format of the input is not the numpy 1-D array;
2) the shape of predictions and actuals does not match;
3) the input n is not a positive integer.
"""
if len(predictions) != len(actuals):
raise ValueError("the shape of predictions and actuals does not match.")
if n is not None:
if not isinstance(n, int) or n <= 0:
raise ValueError("n must be 'None' or a positive integer."
" It was '%s'." % n)
ap = 0.0
predictions = numpy.array(predictions)
actuals = numpy.array(actuals)
# add a shuffler to avoid overestimating the ap
predictions, actuals = AveragePrecisionCalculator._shuffle(predictions,
actuals)
sortidx = sorted(
range(len(predictions)),
key=lambda k: predictions[k],
reverse=True)
if total_num_positives is None:
numpos = numpy.size(numpy.where(actuals > 0))
else:
numpos = total_num_positives
if numpos == 0:
return 0
if n is not None:
numpos = min(numpos, n)
delta_recall = 1.0 / numpos
poscount = 0.0
# calculate the ap
r = len(sortidx)
if n is not None:
r = min(r, n)
for i in range(r):
if actuals[sortidx[i]] > 0:
poscount += 1
ap += poscount / (i + 1) * delta_recall
return ap
@staticmethod
def _shuffle(predictions, actuals):
random.seed(0)
suffidx = random.sample(range(len(predictions)), len(predictions))
predictions = predictions[suffidx]
actuals = actuals[suffidx]
return predictions, actuals
@staticmethod
def _zero_one_normalize(predictions, epsilon=1e-7):
"""Normalize the predictions to the range between 0.0 and 1.0.
For some predictions like SVM predictions, we need to normalize them before
calculate the interpolated average precision. The normalization will not
change the rank in the original list and thus won't change the average
precision.
Args:
predictions: a numpy 1-D array storing the sparse prediction scores.
epsilon: a small constant to avoid denominator being zero.
Returns:
The normalized prediction.
"""
denominator = numpy.max(predictions) - numpy.min(predictions)
ret = (predictions - numpy.min(predictions)) / numpy.max(denominator,
epsilon)
return ret
================================================
FILE: youtube-8m-zhangteng/cloudml-gpu-distributed.yaml
================================================
trainingInput:
runtimeVersion: "1.0"
scaleTier: CUSTOM
masterType: standard_gpu
workerCount: 2
workerType: standard_gpu
parameterServerCount: 2
parameterServerType: standard
================================================
FILE: youtube-8m-zhangteng/cloudml-gpu.yaml
================================================
trainingInput:
scaleTier: CUSTOM
# standard_gpu provides 1 GPU. Change to complex_model_m_gpu for 4 GPUs
masterType: standard_gpu
runtimeVersion: "1.0"
================================================
FILE: youtube-8m-zhangteng/eval.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for evaluating Tensorflow models on the YouTube-8M dataset."""
import time
import eval_util
import losses
import frame_level_models
import video_level_models
import readers
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import utils
FLAGS = flags.FLAGS
if __name__ == "__main__":
# Dataset flags.
flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
"The directory to load the model files from. "
"The tensorboard metrics files are also saved to this "
"directory.")
flags.DEFINE_string("model_checkpoint_path", "",
"The file to load the model files from. ")
flags.DEFINE_string(
"eval_data_pattern", "",
"File glob defining the evaluation dataset in tensorflow.SequenceExample "
"format. The SequenceExamples are expected to have an 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024", "Length of the feature vectors.")
# Model flags.
flags.DEFINE_bool(
"frame_features", False,
"If set, then --eval_data_pattern must be frame-level features. "
"Otherwise, --eval_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_bool(
"norm", True,
"If set, then --input_data should be l2-normalized before follow-up processing. "
"Otherwise, --input_data remain unchanged")
flags.DEFINE_string(
"model", "LogisticModel",
"Which architecture to use for the model. Options include 'Logistic', "
"'SingleMixtureMoe', and 'TwoLayerSigmoid'. See aggregated_models.py and "
"frame_level_models.py for the model definitions.")
flags.DEFINE_integer("batch_size", 1024,
"How many examples to process per batch.")
flags.DEFINE_integer("stride_size", 4,
"How many frames to skip in frame level models, "
"only used in LstmFramesModel and LstmSoftmaxModel")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Loss computed on validation data")
# Other flags.
flags.DEFINE_integer("num_readers", 8,
"How many threads to use for reading input files.")
flags.DEFINE_boolean("run_once", False, "Whether to run eval only once.")
flags.DEFINE_integer("top_k", 20, "How many predictions to output per video.")
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def get_input_evaluation_tensors(reader,
data_pattern,
batch_size=1024,
num_readers=1):
"""Creates the section of the graph which reads the evaluation data.
Args:
reader: A class which parses the training data.
data_pattern: A 'glob' style path to the data files.
batch_size: How many examples to process at a time.
num_readers: How many I/O threads to use.
Returns:
A tuple containing the features tensor, labels tensor, and optionally a
tensor containing the number of frames per video. The exact dimensions
depend on the reader being used.
Raises:
IOError: If no files matching the given pattern were found.
"""
logging.info("Using batch size of " + str(batch_size) + " for evaluation.")
with tf.name_scope("eval_input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find the evaluation files.")
logging.info("number of evaluation files: " + str(len(files)))
filename_queue = tf.train.string_input_producer(
files, shuffle=False, num_epochs=1)
eval_data = [
reader.prepare_reader(filename_queue) for _ in range(num_readers)
]
return tf.train.batch_join(
eval_data,
batch_size=batch_size,
capacity=3 * batch_size,
allow_smaller_final_batch=True,
enqueue_many=True)
def build_graph(reader,
model,
eval_data_pattern,
label_loss_fn,
batch_size=1024,
num_readers=1):
"""Creates the Tensorflow graph for evaluation.
Args:
reader: The data file reader. It should inherit from BaseReader.
model: The core model (e.g. logistic or neural net). It should inherit
from BaseModel.
eval_data_pattern: glob path to the evaluation data files.
label_loss_fn: What kind of loss to apply to the model. It should inherit
from BaseLoss.
batch_size: How many examples to process at a time.
num_readers: How many threads to use for I/O operations.
"""
global_step = tf.Variable(0, trainable=False, name="global_step")
video_id_batch, model_input_raw, labels_batch, num_frames = get_input_evaluation_tensors( # pylint: disable=g-line-too-long
reader,
eval_data_pattern,
batch_size=batch_size,
num_readers=num_readers)
tf.summary.histogram("model_input_raw", model_input_raw)
feature_dim = len(model_input_raw.get_shape()) - 1
# Normalize input features.
if FLAGS.norm:
model_input = tf.nn.l2_normalize(model_input_raw, feature_dim)
else:
model_input = model_input_raw
with tf.name_scope("model"):
result = model.create_model(model_input,
num_frames=num_frames,
vocab_size=reader.num_classes,
labels=labels_batch,
is_training=False)
predictions = result["predictions"]
tf.summary.histogram("model_activations", predictions)
if "loss" in result.keys():
label_loss = result["loss"]
else:
label_loss = label_loss_fn.calculate_loss(predictions, labels_batch)
tf.add_to_collection("global_step", global_step)
tf.add_to_collection("loss", label_loss)
tf.add_to_collection("predictions", predictions)
tf.add_to_collection("input_batch", model_input)
tf.add_to_collection("video_id_batch", video_id_batch)
tf.add_to_collection("num_frames", num_frames)
tf.add_to_collection("labels", tf.cast(labels_batch, tf.float32))
tf.add_to_collection("summary_op", tf.summary.merge_all())
def evaluation_loop(video_id_batch, prediction_batch, label_batch, loss,
summary_op, saver, summary_writer, evl_metrics,
last_global_step_val):
"""Run the evaluation loop once.
Args:
video_id_batch: a tensor of video ids mini-batch.
prediction_batch: a tensor of predictions mini-batch.
label_batch: a tensor of label_batch mini-batch.
loss: a tensor of loss for the examples in the mini-batch.
summary_op: a tensor which runs the tensorboard summary operations.
saver: a tensorflow saver to restore the model.
summary_writer: a tensorflow summary_writer
evl_metrics: an EvaluationMetrics object.
last_global_step_val: the global step used in the previous evaluation.
Returns:
The global_step used in the latest model.
"""
global_step_val = -1
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.3)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
if FLAGS.model_checkpoint_path:
checkpoint = FLAGS.model_checkpoint_path
else:
checkpoint = tf.train.latest_checkpoint(FLAGS.train_dir)
if checkpoint:
logging.info("Loading checkpoint for eval: " + checkpoint)
# Restores from checkpoint
saver.restore(sess, checkpoint)
# Assuming model_checkpoint_path looks something like:
# /my-favorite-path/yt8m_train/model.ckpt-0, extract global_step from it.
global_step_val = checkpoint.split("/")[-1].split("-")[-1]
else:
logging.info("No checkpoint file found.")
return global_step_val
if global_step_val == last_global_step_val:
logging.info("skip this checkpoint global_step_val=%s "
"(same as the previous one).", global_step_val)
return global_step_val
sess.run([tf.local_variables_initializer()])
# Start the queue runners.
fetches = [video_id_batch, prediction_batch, label_batch, loss, summary_op]
coord = tf.train.Coordinator()
try:
threads = []
for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
threads.extend(qr.create_threads(
sess, coord=coord, daemon=True,
start=True))
logging.info("enter eval_once loop global_step_val = %s. ",
global_step_val)
evl_metrics.clear()
examples_processed = 0
while not coord.should_stop():
batch_start_time = time.time()
_, predictions_val, labels_val, loss_val, summary_val = sess.run(
fetches)
seconds_per_batch = time.time() - batch_start_time
example_per_second = labels_val.shape[0] / seconds_per_batch
examples_processed += labels_val.shape[0]
iteration_info_dict = evl_metrics.accumulate(predictions_val,
labels_val, loss_val)
iteration_info_dict["examples_per_second"] = example_per_second
iterinfo = utils.AddGlobalStepSummary(
summary_writer,
global_step_val,
iteration_info_dict,
summary_scope="Eval")
logging.info("examples_processed: %d | %s", examples_processed,
iterinfo)
except tf.errors.OutOfRangeError as e:
logging.info(
"Done with batched inference. Now calculating global performance "
"metrics.")
# calculate the metrics for the entire epoch
epoch_info_dict = evl_metrics.get()
epoch_info_dict["epoch_id"] = global_step_val
summary_writer.add_summary(summary_val, global_step_val)
epochinfo = utils.AddEpochSummary(
summary_writer,
global_step_val,
epoch_info_dict,
summary_scope="Eval")
logging.info(epochinfo)
evl_metrics.clear()
except Exception as e: # pylint: disable=broad-except
logging.info("Unexpected exception: " + str(e))
coord.request_stop(e)
coord.request_stop()
coord.join(threads, stop_grace_period_secs=10)
return global_step_val
def evaluate():
tf.set_random_seed(0) # for reproducibility
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
model = find_class_by_name(FLAGS.model,
[frame_level_models, video_level_models])()
label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])()
if FLAGS.eval_data_pattern is "":
raise IOError("'eval_data_pattern' was not specified. " +
"Nothing to evaluate.")
build_graph(
reader=reader,
model=model,
eval_data_pattern=FLAGS.eval_data_pattern,
label_loss_fn=label_loss_fn,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
video_id_batch = tf.get_collection("video_id_batch")[0]
prediction_batch = tf.get_collection("predictions")[0]
label_batch = tf.get_collection("labels")[0]
loss = tf.get_collection("loss")[0]
summary_op = tf.get_collection("summary_op")[0]
saver = tf.train.Saver(tf.global_variables())
summary_writer = tf.summary.FileWriter(
FLAGS.train_dir, graph=tf.get_default_graph())
evl_metrics = eval_util.EvaluationMetrics(reader.num_classes, FLAGS.top_k)
last_global_step_val = -1
while True:
last_global_step_val = evaluation_loop(video_id_batch, prediction_batch,
label_batch, loss, summary_op,
saver, summary_writer, evl_metrics,
last_global_step_val)
if FLAGS.run_once:
break
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
evaluate()
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-zhangteng/eval_autoencoder.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for evaluating Tensorflow models on the YouTube-8M dataset."""
import time
import eval_util
import losses
import labels_autoencoder
import readers
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import utils
FLAGS = flags.FLAGS
if __name__ == "__main__":
# Dataset flags.
flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
"The directory to load the model files from. "
"The tensorboard metrics files are also saved to this "
"directory.")
flags.DEFINE_string("model_checkpoint_path", "",
"The file to load the model files from. ")
flags.DEFINE_string(
"eval_data_pattern", "",
"File glob defining the evaluation dataset in tensorflow.SequenceExample "
"format. The SequenceExamples are expected to have an 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024", "Length of the feature vectors.")
# Model flags.
flags.DEFINE_bool(
"frame_features", False,
"If set, then --eval_data_pattern must be frame-level features. "
"Otherwise, --eval_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_string(
"model", "LogisticModel",
"Which architecture to use for the model. Options include 'Logistic', "
"'SingleMixtureMoe', and 'TwoLayerSigmoid'. See aggregated_models.py and "
"frame_level_models.py for the model definitions.")
flags.DEFINE_integer("batch_size", 1024,
"How many examples to process per batch.")
flags.DEFINE_integer("stride_size", 4,
"How many examples to process per batch for training.")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Loss computed on validation data")
# Other flags.
flags.DEFINE_integer("num_readers", 8,
"How many threads to use for reading input files.")
flags.DEFINE_boolean("run_once", False, "Whether to run eval only once.")
flags.DEFINE_integer("top_k", 20, "How many predictions to output per video.")
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def get_input_evaluation_tensors(reader,
data_pattern,
batch_size=1024,
num_readers=1):
"""Creates the section of the graph which reads the evaluation data.
Args:
reader: A class which parses the training data.
data_pattern: A 'glob' style path to the data files.
batch_size: How many examples to process at a time.
num_readers: How many I/O threads to use.
Returns:
A tuple containing the features tensor, labels tensor, and optionally a
tensor containing the number of frames per video. The exact dimensions
depend on the reader being used.
Raises:
IOError: If no files matching the given pattern were found.
"""
logging.info("Using batch size of " + str(batch_size) + " for evaluation.")
with tf.name_scope("eval_input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find the evaluation files.")
logging.info("number of evaluation files: " + str(len(files)))
filename_queue = tf.train.string_input_producer(
files, shuffle=False, num_epochs=1)
eval_data = [
reader.prepare_reader(filename_queue) for _ in range(num_readers)
]
return tf.train.batch_join(
eval_data,
batch_size=batch_size,
capacity=3 * batch_size,
allow_smaller_final_batch=True,
enqueue_many=True)
def build_graph(reader,
model,
eval_data_pattern,
label_loss_fn,
batch_size=1024,
num_readers=1):
"""Creates the Tensorflow graph for evaluation.
Args:
reader: The data file reader. It should inherit from BaseReader.
model: The core model (e.g. logistic or neural net). It should inherit
from BaseModel.
eval_data_pattern: glob path to the evaluation data files.
label_loss_fn: What kind of loss to apply to the model. It should inherit
from BaseLoss.
batch_size: How many examples to process at a time.
num_readers: How many threads to use for I/O operations.
"""
global_step = tf.Variable(0, trainable=False, name="global_step")
video_id_batch, model_input_raw, labels_batch, num_frames = get_input_evaluation_tensors( # pylint: disable=g-line-too-long
reader,
eval_data_pattern,
batch_size=batch_size,
num_readers=num_readers)
tf.summary.histogram("model_input_raw", model_input_raw)
with tf.name_scope("model"):
result = model.create_model(labels_batch,
num_frames=num_frames,
vocab_size=reader.num_classes,
labels=labels_batch,
is_training=False)
predictions = result["predictions"]
tf.summary.histogram("model_activations", predictions)
if "loss" in result.keys():
label_loss = result["loss"]
else:
label_loss = label_loss_fn.calculate_loss(predictions, labels_batch)
tf.add_to_collection("global_step", global_step)
tf.add_to_collection("loss", label_loss)
tf.add_to_collection("predictions", predictions)
tf.add_to_collection("video_id_batch", video_id_batch)
tf.add_to_collection("num_frames", num_frames)
tf.add_to_collection("labels", tf.cast(labels_batch, tf.float32))
tf.add_to_collection("summary_op", tf.summary.merge_all())
def evaluation_loop(video_id_batch, prediction_batch, label_batch, loss,
summary_op, saver, summary_writer, evl_metrics,
last_global_step_val):
"""Run the evaluation loop once.
Args:
video_id_batch: a tensor of video ids mini-batch.
prediction_batch: a tensor of predictions mini-batch.
label_batch: a tensor of label_batch mini-batch.
loss: a tensor of loss for the examples in the mini-batch.
summary_op: a tensor which runs the tensorboard summary operations.
saver: a tensorflow saver to restore the model.
summary_writer: a tensorflow summary_writer
evl_metrics: an EvaluationMetrics object.
last_global_step_val: the global step used in the previous evaluation.
Returns:
The global_step used in the latest model.
"""
global_step_val = -1
with tf.Session() as sess:
if FLAGS.model_checkpoint_path:
checkpoint = FLAGS.model_checkpoint_path
else:
checkpoint = tf.train.latest_checkpoint(FLAGS.train_dir)
if checkpoint:
logging.info("Loading checkpoint for eval: " + checkpoint)
# Restores from checkpoint
saver.restore(sess, checkpoint)
# Assuming model_checkpoint_path looks something like:
# /my-favorite-path/yt8m_train/model.ckpt-0, extract global_step from it.
global_step_val = checkpoint.split("/")[-1].split("-")[-1]
else:
logging.info("No checkpoint file found.")
return global_step_val
if global_step_val == last_global_step_val:
logging.info("skip this checkpoint global_step_val=%s "
"(same as the previous one).", global_step_val)
return global_step_val
sess.run([tf.local_variables_initializer()])
# Start the queue runners.
fetches = [video_id_batch, prediction_batch, label_batch, loss, summary_op]
coord = tf.train.Coordinator()
try:
threads = []
for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
threads.extend(qr.create_threads(
sess, coord=coord, daemon=True,
start=True))
logging.info("enter eval_once loop global_step_val = %s. ",
global_step_val)
evl_metrics.clear()
examples_processed = 0
while not coord.should_stop():
batch_start_time = time.time()
_, predictions_val, labels_val, loss_val, summary_val = sess.run(
fetches)
seconds_per_batch = time.time() - batch_start_time
example_per_second = labels_val.shape[0] / seconds_per_batch
examples_processed += labels_val.shape[0]
iteration_info_dict = evl_metrics.accumulate(predictions_val,
labels_val, loss_val)
iteration_info_dict["examples_per_second"] = example_per_second
iterinfo = utils.AddGlobalStepSummary(
summary_writer,
global_step_val,
iteration_info_dict,
summary_scope="Eval")
logging.info("examples_processed: %d | %s", examples_processed,
iterinfo)
except tf.errors.OutOfRangeError as e:
logging.info(
"Done with batched inference. Now calculating global performance "
"metrics.")
# calculate the metrics for the entire epoch
epoch_info_dict = evl_metrics.get()
epoch_info_dict["epoch_id"] = global_step_val
summary_writer.add_summary(summary_val, global_step_val)
epochinfo = utils.AddEpochSummary(
summary_writer,
global_step_val,
epoch_info_dict,
summary_scope="Eval")
logging.info(epochinfo)
evl_metrics.clear()
except Exception as e: # pylint: disable=broad-except
logging.info("Unexpected exception: " + str(e))
coord.request_stop(e)
coord.request_stop()
coord.join(threads, stop_grace_period_secs=10)
return global_step_val
def evaluate():
tf.set_random_seed(0) # for reproducibility
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
model = find_class_by_name(FLAGS.model,
[labels_autoencoder])()
label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])()
if FLAGS.eval_data_pattern is "":
raise IOError("'eval_data_pattern' was not specified. " +
"Nothing to evaluate.")
build_graph(
reader=reader,
model=model,
eval_data_pattern=FLAGS.eval_data_pattern,
label_loss_fn=label_loss_fn,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
video_id_batch = tf.get_collection("video_id_batch")[0]
prediction_batch = tf.get_collection("predictions")[0]
label_batch = tf.get_collection("labels")[0]
loss = tf.get_collection("loss")[0]
summary_op = tf.get_collection("summary_op")[0]
saver = tf.train.Saver(tf.global_variables())
summary_writer = tf.summary.FileWriter(
FLAGS.train_dir, graph=tf.get_default_graph())
evl_metrics = eval_util.EvaluationMetrics(reader.num_classes, FLAGS.top_k)
last_global_step_val = -1
while True:
last_global_step_val = evaluation_loop(video_id_batch, prediction_batch,
label_batch, loss, summary_op,
saver, summary_writer, evl_metrics,
last_global_step_val)
if FLAGS.run_once:
break
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
evaluate()
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-zhangteng/eval_distill.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for evaluating Tensorflow models on the YouTube-8M dataset."""
import time
import eval_util
import losses
import frame_level_models
import video_level_models
import readers
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import utils
FLAGS = flags.FLAGS
if __name__ == "__main__":
# Dataset flags.
flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
"The directory to load the model files from. "
"The tensorboard metrics files are also saved to this "
"directory.")
flags.DEFINE_string("model_checkpoint_path", "",
"The file to load the model files from. ")
flags.DEFINE_string(
"eval_data_pattern", "",
"File glob defining the evaluation dataset in tensorflow.SequenceExample "
"format. The SequenceExamples are expected to have an 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024", "Length of the feature vectors.")
flags.DEFINE_string(
"distill_data_pattern", "",
"File glob defining the distill dataset in tensorflow.SequenceExample "
"format. The SequenceExamples are expected to have an 'predictions' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
flags.DEFINE_string("distill_names", "predictions", "Name of the distill feature "
"to use for eval.")
flags.DEFINE_string("distill_sizes", "4716", "Length of the distill feature vectors.")
# Model flags.
flags.DEFINE_bool(
"frame_features", False,
"If set, then --eval_data_pattern must be frame-level features. "
"Otherwise, --eval_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_bool(
"norm", True,
"If set, then --input_data should be l2-normalized before follow-up processing. "
"Otherwise, --input_data remain unchanged")
flags.DEFINE_string(
"model", "LogisticModel",
"Which architecture to use for the model. Options include 'Logistic', "
"'SingleMixtureMoe', and 'TwoLayerSigmoid'. See aggregated_models.py and "
"frame_level_models.py for the model definitions.")
flags.DEFINE_integer("batch_size", 1024,
"How many examples to process per batch.")
flags.DEFINE_integer("stride_size", 4,
"How many frames to skip in frame level models, "
"only used in LstmFramesModel and LstmSoftmaxModel")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Loss computed on validation data")
# Other flags.
flags.DEFINE_integer("num_readers", 2,
"How many threads to use for reading input files.")
flags.DEFINE_boolean("run_once", False, "Whether to run eval only once.")
flags.DEFINE_integer("top_k", 20, "How many predictions to output per video.")
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def get_input_evaluation_tensors(reader,
data_pattern,
batch_size=1024,
num_readers=1):
"""Creates the section of the graph which reads the evaluation data.
Args:
reader: A class which parses the training data.
data_pattern: A 'glob' style path to the data files.
batch_size: How many examples to process at a time.
num_readers: How many I/O threads to use.
Returns:
A tuple containing the features tensor, labels tensor, and optionally a
tensor containing the number of frames per video. The exact dimensions
depend on the reader being used.
Raises:
IOError: If no files matching the given pattern were found.
"""
logging.info("Using batch size of " + str(batch_size) + " for evaluation.")
with tf.name_scope("eval_input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find the evaluation files.")
logging.info("number of evaluation files: " + str(len(files)))
files.sort()
filename_queue = tf.train.string_input_producer(
files, shuffle=False, num_epochs=1)
eval_data = reader.prepare_reader(filename_queue)
return tf.train.batch(
eval_data,
batch_size=batch_size,
capacity=3 * batch_size,
allow_smaller_final_batch=True,
enqueue_many=True)
def build_graph(reader1,
reader2,
model,
eval_data_pattern,
distill_data_pattern,
label_loss_fn,
batch_size=1024,
num_readers=1):
"""Creates the Tensorflow graph for evaluation.
Args:
reader: The data file reader. It should inherit from BaseReader.
model: The core model (e.g. logistic or neural net). It should inherit
from BaseModel.
eval_data_pattern: glob path to the evaluation data files.
label_loss_fn: What kind of loss to apply to the model. It should inherit
from BaseLoss.
batch_size: How many examples to process at a time.
num_readers: How many threads to use for I/O operations.
"""
global_step = tf.Variable(0, trainable=False, name="global_step")
video_id_batch, model_input_raw, labels_batch, num_frames = get_input_evaluation_tensors( # pylint: disable=g-line-too-long
reader1,
eval_data_pattern,
batch_size=batch_size,
num_readers=num_readers)
unused_id_batch, labels_distill, unused_labels_batch, unusednum_frames = get_input_evaluation_tensors( # pylint: disable=g-line-too-long
reader2,
distill_data_pattern,
batch_size=batch_size,
num_readers=num_readers)
tf.summary.histogram("model_input_raw", model_input_raw)
feature_dim = len(model_input_raw.get_shape()) - 1
# Normalize input features.
if FLAGS.norm:
model_input = tf.nn.l2_normalize(model_input_raw, feature_dim)
else:
model_input = model_input_raw
with tf.name_scope("model"):
result = model.create_model(model_input,
num_frames=num_frames,
vocab_size=reader1.num_classes,
labels=labels_batch,
distill_labels=labels_distill,
is_training=False)
predictions = result["predictions"]
#predictions = 0.0*predictions + 1.0*labels_distill
tf.summary.histogram("model_activations", predictions)
if "loss" in result.keys():
label_loss = result["loss"]
else:
label_loss = label_loss_fn.calculate_loss(predictions, labels_batch)
tf.add_to_collection("global_step", global_step)
tf.add_to_collection("loss", label_loss)
tf.add_to_collection("predictions", predictions)
tf.add_to_collection("input_batch", model_input)
tf.add_to_collection("video_id_batch", video_id_batch)
tf.add_to_collection("unused_id_batch", unused_id_batch)
tf.add_to_collection("num_frames", num_frames)
tf.add_to_collection("labels", tf.cast(labels_batch, tf.float32))
tf.add_to_collection("summary_op", tf.summary.merge_all())
def evaluation_loop(video_id_batch, unused_id_batch, prediction_batch, label_batch, loss,
summary_op, saver, summary_writer, evl_metrics,
last_global_step_val):
"""Run the evaluation loop once.
Args:
video_id_batch: a tensor of video ids mini-batch.
prediction_batch: a tensor of predictions mini-batch.
label_batch: a tensor of label_batch mini-batch.
loss: a tensor of loss for the examples in the mini-batch.
summary_op: a tensor which runs the tensorboard summary operations.
saver: a tensorflow saver to restore the model.
summary_writer: a tensorflow summary_writer
evl_metrics: an EvaluationMetrics object.
last_global_step_val: the global step used in the previous evaluation.
Returns:
The global_step used in the latest model.
"""
global_step_val = -1
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.3)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
if FLAGS.model_checkpoint_path:
checkpoint = FLAGS.model_checkpoint_path
else:
checkpoint = tf.train.latest_checkpoint(FLAGS.train_dir)
if checkpoint:
logging.info("Loading checkpoint for eval: " + checkpoint)
# Restores from checkpoint
saver.restore(sess, checkpoint)
# Assuming model_checkpoint_path looks something like:
# /my-favorite-path/yt8m_train/model.ckpt-0, extract global_step from it.
global_step_val = checkpoint.split("/")[-1].split("-")[-1]
else:
logging.info("No checkpoint file found.")
return global_step_val
if global_step_val == last_global_step_val:
logging.info("skip this checkpoint global_step_val=%s "
"(same as the previous one).", global_step_val)
return global_step_val
sess.run([tf.local_variables_initializer()])
# Start the queue runners.
fetches = [video_id_batch, unused_id_batch, prediction_batch, label_batch, loss, summary_op]
coord = tf.train.Coordinator()
try:
threads = []
for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
threads.extend(qr.create_threads(
sess, coord=coord, daemon=True,
start=True))
logging.info("enter eval_once loop global_step_val = %s. ",
global_step_val)
evl_metrics.clear()
examples_processed = 0
while not coord.should_stop():
batch_start_time = time.time()
eval_id, distill_id, predictions_val, labels_val, loss_val, summary_val = sess.run(
fetches)
seconds_per_batch = time.time() - batch_start_time
example_per_second = labels_val.shape[0] / seconds_per_batch
examples_processed += labels_val.shape[0]
iteration_info_dict = evl_metrics.accumulate(predictions_val,
labels_val, loss_val)
iteration_info_dict["examples_per_second"] = example_per_second
iterinfo = utils.AddGlobalStepSummary(
summary_writer,
global_step_val,
iteration_info_dict,
summary_scope="Eval")
logging.info("examples_processed: %d | %s", examples_processed,
iterinfo)
"""
if examples_processed > 22000:
logging.info(
"Done with batched inference. Now calculating global performance "
"metrics.")
# calculate the metrics for the entire epoch
epoch_info_dict = evl_metrics.get()
epoch_info_dict["epoch_id"] = global_step_val
summary_writer.add_summary(summary_val, global_step_val)
epochinfo = utils.AddEpochSummary(
summary_writer,
global_step_val,
epoch_info_dict,
summary_scope="Eval")
logging.info(epochinfo)
evl_metrics.clear()
break"""
except tf.errors.OutOfRangeError as e:
logging.info(
"Done with batched inference. Now calculating global performance "
"metrics.")
# calculate the metrics for the entire epoch
epoch_info_dict = evl_metrics.get()
epoch_info_dict["epoch_id"] = global_step_val
summary_writer.add_summary(summary_val, global_step_val)
epochinfo = utils.AddEpochSummary(
summary_writer,
global_step_val,
epoch_info_dict,
summary_scope="Eval")
logging.info(epochinfo)
evl_metrics.clear()
except Exception as e: # pylint: disable=broad-except
logging.info("Unexpected exception: " + str(e))
coord.request_stop(e)
coord.request_stop()
coord.join(threads, stop_grace_period_secs=10)
return global_step_val
def evaluate():
tf.set_random_seed(0) # for reproducibility
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
distill_names, distill_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.distill_names, FLAGS.distill_sizes)
if FLAGS.frame_features:
reader1 = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader1 = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
reader2 = readers.YT8MAggregatedFeatureReader(
feature_names=distill_names, feature_sizes=distill_sizes)
model = find_class_by_name(FLAGS.model,
[frame_level_models, video_level_models])()
label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])()
if FLAGS.eval_data_pattern is "":
raise IOError("'eval_data_pattern' was not specified. " +
"Nothing to evaluate.")
build_graph(
reader1=reader1,
reader2=reader2,
model=model,
eval_data_pattern=FLAGS.eval_data_pattern,
distill_data_pattern=FLAGS.distill_data_pattern,
label_loss_fn=label_loss_fn,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
video_id_batch = tf.get_collection("video_id_batch")[0]
unused_id_batch = tf.get_collection("unused_id_batch")[0]
prediction_batch = tf.get_collection("predictions")[0]
label_batch = tf.get_collection("labels")[0]
loss = tf.get_collection("loss")[0]
summary_op = tf.get_collection("summary_op")[0]
saver = tf.train.Saver(tf.global_variables())
summary_writer = tf.summary.FileWriter(
FLAGS.train_dir, graph=tf.get_default_graph())
evl_metrics = eval_util.EvaluationMetrics(reader1.num_classes, FLAGS.top_k)
last_global_step_val = -1
while True:
last_global_step_val = evaluation_loop(video_id_batch, unused_id_batch, prediction_batch,
label_batch, loss, summary_op,
saver, summary_writer, evl_metrics,
last_global_step_val)
if FLAGS.run_once:
break
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
evaluate()
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-zhangteng/eval_embedding.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for evaluating Tensorflow models on the YouTube-8M dataset."""
import time
import eval_util
import losses
import labels_autoencoder
import readers
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import utils
FLAGS = flags.FLAGS
if __name__ == "__main__":
# Dataset flags.
flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
"The directory to load the model files from. "
"The tensorboard metrics files are also saved to this "
"directory.")
flags.DEFINE_string("model_checkpoint_path", "",
"The file to load the model files from. ")
flags.DEFINE_string(
"eval_data_pattern", "",
"File glob defining the evaluation dataset in tensorflow.SequenceExample "
"format. The SequenceExamples are expected to have an 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024", "Length of the feature vectors.")
# Model flags.
flags.DEFINE_bool(
"frame_features", False,
"If set, then --eval_data_pattern must be frame-level features. "
"Otherwise, --eval_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_string(
"model", "LogisticModel",
"Which architecture to use for the model. Options include 'Logistic', "
"'SingleMixtureMoe', and 'TwoLayerSigmoid'. See aggregated_models.py and "
"frame_level_models.py for the model definitions.")
flags.DEFINE_integer("batch_size", 1024,
"How many examples to process per batch.")
flags.DEFINE_integer("stride_size", 4,
"How many frames to skip in frame level models, "
"only used in LstmFramesModel and LstmSoftmaxModel")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Loss computed on validation data")
# Other flags.
flags.DEFINE_integer("num_readers", 8,
"How many threads to use for reading input files.")
flags.DEFINE_boolean("run_once", False, "Whether to run eval only once.")
flags.DEFINE_integer("top_k", 20, "How many predictions to output per video.")
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def get_input_evaluation_tensors(reader,
data_pattern,
batch_size=1024,
num_readers=1):
"""Creates the section of the graph which reads the evaluation data.
Args:
reader: A class which parses the training data.
data_pattern: A 'glob' style path to the data files.
batch_size: How many examples to process at a time.
num_readers: How many I/O threads to use.
Returns:
A tuple containing the features tensor, labels tensor, and optionally a
tensor containing the number of frames per video. The exact dimensions
depend on the reader being used.
Raises:
IOError: If no files matching the given pattern were found.
"""
logging.info("Using batch size of " + str(batch_size) + " for evaluation.")
with tf.name_scope("eval_input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find the evaluation files.")
logging.info("number of evaluation files: " + str(len(files)))
filename_queue = tf.train.string_input_producer(
files, shuffle=False, num_epochs=1)
eval_data = [
reader.prepare_reader(filename_queue) for _ in range(num_readers)
]
return tf.train.batch_join(
eval_data,
batch_size=batch_size,
capacity=3 * batch_size,
allow_smaller_final_batch=True,
enqueue_many=True)
def build_graph(reader,
model,
eval_data_pattern,
label_loss_fn,
batch_size=1024,
num_readers=1):
"""Creates the Tensorflow graph for evaluation.
Args:
reader: The data file reader. It should inherit from BaseReader.
model: The core model (e.g. logistic or neural net). It should inherit
from BaseModel.
eval_data_pattern: glob path to the evaluation data files.
label_loss_fn: What kind of loss to apply to the model. It should inherit
from BaseLoss.
batch_size: How many examples to process at a time.
num_readers: How many threads to use for I/O operations.
"""
global_step = tf.Variable(0, trainable=False, name="global_step")
video_id_batch, model_input_raw, labels_batch, num_frames = get_input_evaluation_tensors( # pylint: disable=g-line-too-long
reader,
eval_data_pattern,
batch_size=batch_size,
num_readers=num_readers)
tf.summary.histogram("model_input_raw", model_input_raw)
with tf.name_scope("model"):
result = model.create_model(labels_batch,
num_frames=num_frames,
vocab_size=reader.num_classes,
labels=labels_batch,
is_training=False)
predictions = result["predictions"]
tf.summary.histogram("model_activations", predictions)
if "loss" in result.keys():
label_loss = result["loss"]
else:
label_loss = label_loss_fn.calculate_loss(predictions, labels_batch)
tf.add_to_collection("global_step", global_step)
tf.add_to_collection("loss", label_loss)
tf.add_to_collection("predictions", predictions)
tf.add_to_collection("video_id_batch", video_id_batch)
tf.add_to_collection("num_frames", num_frames)
tf.add_to_collection("labels", tf.cast(labels_batch, tf.float32))
tf.add_to_collection("summary_op", tf.summary.merge_all())
def evaluation_loop(video_id_batch, prediction_batch, label_batch, loss,
summary_op, saver, summary_writer, evl_metrics,
last_global_step_val):
"""Run the evaluation loop once.
Args:
video_id_batch: a tensor of video ids mini-batch.
prediction_batch: a tensor of predictions mini-batch.
label_batch: a tensor of label_batch mini-batch.
loss: a tensor of loss for the examples in the mini-batch.
summary_op: a tensor which runs the tensorboard summary operations.
saver: a tensorflow saver to restore the model.
summary_writer: a tensorflow summary_writer
evl_metrics: an EvaluationMetrics object.
last_global_step_val: the global step used in the previous evaluation.
Returns:
The global_step used in the latest model.
"""
global_step_val = -1
with tf.Session() as sess:
if FLAGS.model_checkpoint_path:
checkpoint = FLAGS.model_checkpoint_path
else:
checkpoint = tf.train.latest_checkpoint(FLAGS.train_dir)
if checkpoint:
logging.info("Loading checkpoint for eval: " + checkpoint)
# Restores from checkpoint
saver.restore(sess, checkpoint)
# Assuming model_checkpoint_path looks something like:
# /my-favorite-path/yt8m_train/model.ckpt-0, extract global_step from it.
global_step_val = checkpoint.split("/")[-1].split("-")[-1]
else:
logging.info("No checkpoint file found.")
return global_step_val
if global_step_val == last_global_step_val:
logging.info("skip this checkpoint global_step_val=%s "
"(same as the previous one).", global_step_val)
return global_step_val
sess.run([tf.local_variables_initializer()])
# Start the queue runners.
fetches = [video_id_batch, prediction_batch, label_batch, loss, summary_op]
coord = tf.train.Coordinator()
try:
threads = []
for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
threads.extend(qr.create_threads(
sess, coord=coord, daemon=True,
start=True))
logging.info("enter eval_once loop global_step_val = %s. ",
global_step_val)
evl_metrics.clear()
examples_processed = 0
while not coord.should_stop():
batch_start_time = time.time()
_, predictions_val, labels_val, loss_val, summary_val = sess.run(
fetches)
seconds_per_batch = time.time() - batch_start_time
example_per_second = labels_val.shape[0] / seconds_per_batch
examples_processed += labels_val.shape[0]
iteration_info_dict = evl_metrics.accumulate(predictions_val,
labels_val, loss_val)
iteration_info_dict["examples_per_second"] = example_per_second
iterinfo = utils.AddGlobalStepSummary(
summary_writer,
global_step_val,
iteration_info_dict,
summary_scope="Eval")
logging.info("examples_processed: %d | %s", examples_processed,
iterinfo)
except tf.errors.OutOfRangeError as e:
logging.info(
"Done with batched inference. Now calculating global performance "
"metrics.")
# calculate the metrics for the entire epoch
epoch_info_dict = evl_metrics.get()
epoch_info_dict["epoch_id"] = global_step_val
summary_writer.add_summary(summary_val, global_step_val)
epochinfo = utils.AddEpochSummary(
summary_writer,
global_step_val,
epoch_info_dict,
summary_scope="Eval")
logging.info(epochinfo)
evl_metrics.clear()
except Exception as e: # pylint: disable=broad-except
logging.info("Unexpected exception: " + str(e))
coord.request_stop(e)
coord.request_stop()
coord.join(threads, stop_grace_period_secs=10)
return global_step_val
def evaluate():
tf.set_random_seed(0) # for reproducibility
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
model = find_class_by_name(FLAGS.model,
[labels_autoencoder])()
label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])()
if FLAGS.eval_data_pattern is "":
raise IOError("'eval_data_pattern' was not specified. " +
"Nothing to evaluate.")
build_graph(
reader=reader,
model=model,
eval_data_pattern=FLAGS.eval_data_pattern,
label_loss_fn=label_loss_fn,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
video_id_batch = tf.get_collection("video_id_batch")[0]
prediction_batch = tf.get_collection("predictions")[0]
label_batch = tf.get_collection("labels")[0]
loss = tf.get_collection("loss")[0]
summary_op = tf.get_collection("summary_op")[0]
saver = tf.train.Saver(tf.global_variables())
summary_writer = tf.summary.FileWriter(
FLAGS.train_dir, graph=tf.get_default_graph())
evl_metrics = eval_util.EvaluationMetrics(reader.num_classes, FLAGS.top_k)
last_global_step_val = -1
while True:
last_global_step_val = evaluation_loop(video_id_batch, prediction_batch,
label_batch, loss, summary_op,
saver, summary_writer, evl_metrics,
last_global_step_val)
if FLAGS.run_once:
break
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
evaluate()
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-zhangteng/eval_scripts/eval-distillchain_cnndcc_layer2moe4.sh
================================================
GPU_ID=1
EVERY=1000
MODEL=CnnDCCDistillChainModel
MODEL_DIR="../model/frame_level_cnndcc_distillchain_model"
EVAL_DIR="../model/frame_level_cnndcc_distillchain_model"
start=0
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [[ $checkpoint -gt $start ]]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval_distill.py \
--train_dir="$EVAL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--distill_data_pattern="/Youtube-8M/model_predictions/validatea/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--batch_size=64 \
--model=$MODEL \
--moe_num_mixtures=4 \
--moe_layers=2 \
--distillation_features=True \
--distillation_type=0 \
--ensemble_w=0.2 \
--train=False \
--run_once=True
fi
done
================================================
FILE: youtube-8m-zhangteng/eval_scripts/eval-distillchain_lstm_extend_moe8.sh
================================================
GPU_ID=1
EVERY=1000
MODEL=LstmExtendModel
MODEL_DIR="../model/frame_level_lstm_extend_distillchain_model"
EVAL_DIR="../model/frame_level_lstm_extend_distillchain_model"
start=0
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [[ $checkpoint -gt $start ]]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval_distill.py \
--train_dir="$EVAL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--distill_data_pattern="/Youtube-8M/model_predictions/validatea/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--batch_size=64 \
--model=$MODEL \
--video_level_classifier_model=MoeExtendDistillChainModel \
--moe_num_extend=8 \
--moe_method=None \
--lstm_cells=1024 \
--class_size=100 \
--encoder_size=100 \
--moe_num_mixtures=8 \
--distillation_features=True \
--distillation_type=0 \
--ensemble_w=0.2 \
--train=False \
--run_once=True
fi
done
================================================
FILE: youtube-8m-zhangteng/eval_scripts/eval-distillchain_lstm_gate_moe8.sh
================================================
GPU_ID=1
EVERY=1000
MODEL=LstmGateModel
MODEL_DIR="../model/frame_level_lstm_gate_distillchain_model"
EVAL_DIR="../model/frame_level_lstm_gate_distillchain_model"
start=0
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [[ $checkpoint -gt $start ]]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval_distill.py \
--train_dir="$EVAL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--distill_data_pattern="/Youtube-8M/model_predictions/validatea/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--batch_size=64 \
--model=$MODEL \
--video_level_classifier_model=MoeDistillChainModel \
--moe_num_extend=8 \
--moe_method=None \
--lstm_cells=1024 \
--moe_num_mixtures=8 \
--distillation_features=True \
--distillation_type=0 \
--ensemble_w=0.2 \
--train=False \
--run_once=True
fi
done
================================================
FILE: youtube-8m-zhangteng/eval_scripts/eval-distillchain_lstm_gate_moe8_v2.sh
================================================
GPU_ID=1
EVERY=1000
MODEL=LstmGateModel
MODEL_DIR="../model/frame_level_lstm_gate_distillchain_v2_model"
EVAL_DIR="../model/frame_level_lstm_gate_distillchain_v2_model"
start=0
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [[ $checkpoint -gt $start ]]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval_distill.py \
--train_dir="$EVAL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--distill_data_pattern="/Youtube-8M/model_predictions/validatea/distillation/ensemble_v2_matrix_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--batch_size=64 \
--model=$MODEL \
--video_level_classifier_model=MoeDistillChainModel \
--moe_num_mixtures=8 \
--distillation_features=True \
--distillation_type=0 \
--ensemble_w=0.2 \
--train=False \
--run_once=True
fi
done
================================================
FILE: youtube-8m-zhangteng/eval_scripts/eval-distillchain_lstm_glu2_moe8_v2.sh
================================================
GPU_ID=1
EVERY=1000
MODEL=LstmGlu2Model
MODEL_DIR="../model/frame_level_lstm_glu2_distillchain_v2_model"
EVAL_DIR="../model/frame_level_lstm_glu2_distillchain_v2_model"
start=0
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [[ $checkpoint -gt $start ]]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval_distill.py \
--train_dir="$EVAL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--distill_data_pattern="/Youtube-8M/model_predictions/validatea/distillation/ensemble_v2_matrix_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--batch_size=64 \
--model=$MODEL \
--video_level_classifier_model=MoeDistillChainModel \
--moe_num_mixtures=8 \
--distillation_features=True \
--distillation_type=0 \
--ensemble_w=0.2 \
--train=False \
--run_once=True
fi
done
================================================
FILE: youtube-8m-zhangteng/eval_scripts/eval-distillchain_lstm_moe8.sh
================================================
GPU_ID=1
EVERY=1000
MODEL=LstmModel
MODEL_DIR="../model/frame_level_lstm_distillchain_model"
EVAL_DIR="../model/frame_level_lstm_distillchain_model"
start=0
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [[ $checkpoint -gt $start ]]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval_distill.py \
--train_dir="$EVAL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--distill_data_pattern="/Youtube-8M/model_predictions/validatea/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--batch_size=64 \
--model=$MODEL \
--video_level_classifier_model=MoeDistillChainModel \
--moe_num_mixtures=8 \
--lstm_layers=1 \
--distillation_features=True \
--distillation_type=0 \
--ensemble_w=0.2 \
--train=False \
--run_once=True
fi
done
================================================
FILE: youtube-8m-zhangteng/eval_scripts/eval-distillchain_lstm_moe8_v2.sh
================================================
GPU_ID=1
EVERY=1000
MODEL=LstmModel
MODEL_DIR="../model/frame_level_lstm_distillchain_v2_model"
EVAL_DIR="../model/frame_level_lstm_distillchain_v2_model"
start=0
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [[ $checkpoint -gt $start ]]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval_distill.py \
--train_dir="$EVAL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--distill_data_pattern="/Youtube-8M/model_predictions/validatea/distillation/ensemble_v2_matrix_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--batch_size=64 \
--model=$MODEL \
--video_level_classifier_model=MoeDistillChainModel \
--moe_num_mixtures=8 \
--lstm_layers=1 \
--distillation_features=True \
--distillation_type=0 \
--ensemble_w=0.2 \
--train=False \
--run_once=True
fi
done
================================================
FILE: youtube-8m-zhangteng/eval_scripts/eval-distillchain_lstm_multiscale2layer_moe8.sh
================================================
GPU_ID=1
EVERY=1000
MODEL=MoeDistillChainModel
MODEL_DIR="../model/frame_level_lstm_multiscale_distillchain_model"
EVAL_DIR="../model/frame_level_lstm_multiscale_distillchain_model"
start=0
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [[ $checkpoint -gt $start ]]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval_distill.py \
--train_dir="$EVAL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--distill_data_pattern="/Youtube-8M/model_predictions/validatea/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--batch_size=64 \
--model=$MODEL \
--moe_num_extend=2 \
--moe_num_mixtures=4 \
--cnn_cells=256 \
--distillation_features=True \
--distillation_type=0 \
--ensemble_w=0.2 \
--train=False \
--run_once=True
fi
done
================================================
FILE: youtube-8m-zhangteng/eval_scripts/eval-distillchain_lstm_multiscale4layer_moe8.sh
================================================
GPU_ID=1
EVERY=1000
MODEL=LstmMultiscaleDitillChainModel
MODEL_DIR="../model/frame_level_lstm_multiscale_distillchain4_model"
EVAL_DIR="../model/frame_level_lstm_multiscale_distillchain4_model"
start=0
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [[ $checkpoint -gt $start ]]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval_distill.py \
--train_dir="$EVAL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--distill_data_pattern="/Youtube-8M/model_predictions/validatea/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--batch_size=64 \
--model=$MODEL \
--moe_num_extend=4 \
--moe_num_mixtures=4 \
--cnn_cells=196 \
--distillation_features=True \
--distillation_type=0 \
--ensemble_w=0.2 \
--train=False \
--run_once=True
fi
done
================================================
FILE: youtube-8m-zhangteng/eval_scripts/eval-distillchain_video_norm_moe8_local.sh
================================================
GPU_ID=0
EVERY=1000
MODEL=MoeDistillChainNorm2Model
MODEL_DIR="../model/video_level_distillchainnorm2_model"
start=0
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [[ $checkpoint -gt $start ]]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval_distill.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--distill_data_pattern="/Youtube-8M/model_predictions/validatea/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--batch_size=64 \
--model=$MODEL \
--moe_num_extend=8 \
--moe_num_mixtures=8 \
--distillation_features=True \
--distillation_type=0 \
--ensemble_w=0.2 \
--noise_std=0.0 \
--train=False \
--run_once=True
fi
done
================================================
FILE: youtube-8m-zhangteng/eval_scripts/eval-distillsplit_lstm_gate_moe8.sh
================================================
GPU_ID=1
EVERY=1000
MODEL=LstmGateModel
MODEL_DIR="../model/frame_level_lstm_gate_distillsplit_model"
start=0
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [[ $checkpoint -gt $start ]]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval_distill.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--distill_data_pattern="/Youtube-8M/model_predictions/validatea/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--batch_size=64 \
--model=$MODEL \
--video_level_classifier_model=MoeDistillSplitModel \
--moe_num_mixtures=8 \
--distillation_features=True \
--distillation_type=0 \
--ensemble_w=1.0 \
--train=False \
--run_once=True
fi
done
================================================
FILE: youtube-8m-zhangteng/eval_scripts/eval-lstm2_attention8_max.sh
================================================
GPU_ID=1
EVERY=1000
MODEL=InputExtendModel
MODEL_DIR="../model/frame_level_input_extend_model"
start=0
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [[ $checkpoint -gt $start ]]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=64 \
--model=$MODEL \
--video_level_classifier_model=MoeExtendModel \
--moe_num_extend=4 \
--moe_num_mixtures=8 \
--train=False \
--run_once=True
fi
done
================================================
FILE: youtube-8m-zhangteng/eval_scripts/eval-lstm_attention8_max.sh
================================================
GPU_ID=1
EVERY=1000
MODEL=LstmExtendModel
MODEL_DIR="../model/frame_level_lstm_extend8_model"
start=0
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [[ $checkpoint -gt $start ]]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=64 \
--model=$MODEL \
--video_level_classifier_model=MoeExtendModel \
--moe_num_extend=8 \
--moe_num_mixtures=8 \
--train=False \
--run_once=True
fi
done
================================================
FILE: youtube-8m-zhangteng/eval_scripts/eval-lstm_gate_multiscale4_moe4.sh
================================================
GPU_ID=1
EVERY=1000
MODEL=LstmMultiscale2Model
MODEL_DIR="../model/frame_level_lstm_multiscale2_model"
start=0
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [[ $checkpoint -gt $start ]]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=64 \
--model=$MODEL \
--video_level_classifier_model=MoeModel \
--norm=False \
--moe_num_extend=4 \
--moe_num_mixtures=4 \
--train=False \
--run_once=True
fi
done
================================================
FILE: youtube-8m-zhangteng/eval_scripts/eval-lstm_multiscale4_moe4.sh
================================================
GPU_ID=1
EVERY=1000
MODEL=LstmMultiscaleModel
MODEL_DIR="../model/frame_level_lstm_multiscale_model"
start=0
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [[ $checkpoint -gt $start ]]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=64 \
--model=$MODEL \
--video_level_classifier_model=MoeModel \
--moe_num_extend=4 \
--moe_num_mixtures=4 \
--train=False \
--run_once=True
fi
done
================================================
FILE: youtube-8m-zhangteng/eval_scripts/eval-lstm_random_moe8.sh
================================================
GPU_ID=1
EVERY=1000
MODEL=LstmRandomModel
MODEL_DIR="../model/frame_level_lstm_random_model"
start=0
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [[ $checkpoint -gt $start ]]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=64 \
--model=$MODEL \
--video_level_classifier_model=MoeModel \
--moe_num_mixtures=8 \
--train=False \
--run_once=True
fi
done
================================================
FILE: youtube-8m-zhangteng/eval_scripts/eval-lstm_shortlayers_moe8.sh
================================================
GPU_ID=1
EVERY=1000
MODEL=LstmLayerModel
MODEL_DIR="../model/frame_level_lstm_layer_model"
start=0
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [[ $checkpoint -gt $start ]]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=64 \
--model=$MODEL \
--video_level_classifier_model=MoeModel \
--lstm_length=10 \
--moe_num_mixtures=8 \
--train=False \
--run_once=True
fi
done
================================================
FILE: youtube-8m-zhangteng/eval_scripts/eval-lstmbiglu_1024_moe8.sh
================================================
GPU_ID=1
EVERY=1000
MODEL=LstmBigluModel
MODEL_DIR="../model/frame_level_lstm_biglu_model"
start=0
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [[ $checkpoint -gt $start ]]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=64 \
--model=$MODEL \
--moe_num_mixtures=4 \
--train=False \
--run_once=True
fi
done
================================================
FILE: youtube-8m-zhangteng/eval_scripts/eval-lstmgate1024_moe8.sh
================================================
GPU_ID=1
EVERY=1000
MODEL=LstmGateModel
MODEL_DIR="../model/frame_level_lstm_gate_model"
start=0
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [[ $checkpoint -gt $start ]]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=64 \
--model=$MODEL \
--video_level_classifier_model=MoeModel \
--moe_num_mixtures=8 \
--train=False \
--run_once=True
fi
done
================================================
FILE: youtube-8m-zhangteng/eval_scripts/eval-lstmglu2_1024_moe8.sh
================================================
GPU_ID=1
EVERY=1000
MODEL=LstmGlu2Model
MODEL_DIR="../model/frame_level_lstm_glu2_model"
start=0
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [[ $checkpoint -gt $start ]]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/frame/validate/validatea*" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=64 \
--model=$MODEL \
--video_level_classifier_model=MoeModel \
--moe_num_mixtures=8 \
--train=False \
--run_once=True
fi
done
================================================
FILE: youtube-8m-zhangteng/eval_scripts/eval-video_knowledge_combine_chain.sh
================================================
GPU_ID=0
EVERY=1000
MODEL=MoeKnowledgeModel
MODEL_DIR="../model/video_level_moeknowledge_model"
start=0
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [[ $checkpoint -gt $start ]]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--model=$MODEL \
--class_size=100 \
--moe_num_mixtures=4 \
--moe_layers=3 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-zhangteng/eval_scripts/eval-video_notzero_combine_chain.sh
================================================
GPU_ID=0
EVERY=1000
MODEL=MoeMix4Model
MODEL_DIR="../model/video_level_moemix4_model"
start=0
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [[ $checkpoint -gt $start ]]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--model=$MODEL \
--class_size=100 \
--moe_num_mixtures=4 \
--moe_layers=3 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-zhangteng/eval_scripts/eval-video_relabel_combine_chain.sh
================================================
GPU_ID=0
EVERY=1000
MODEL=MoeMix4Model
MODEL_DIR="../model/video_level_moemix4_relabel_model"
start=0
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [[ $checkpoint -gt $start ]]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--model=$MODEL \
--class_size=100 \
--moe_num_mixtures=4 \
--moe_layers=3 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-zhangteng/eval_scripts/eval-video_softmax_combine_chain.sh
================================================
GPU_ID=0
EVERY=1000
MODEL=MoeSoftmaxModel
MODEL_DIR="../model/video_level_moesoftmax_model"
start=0
DIR="$(pwd)"
for checkpoint in $(cd $MODEL_DIR && python ${DIR}/training_utils/select.py $EVERY); do
echo $checkpoint;
if [[ $checkpoint -gt $start ]]; then
echo $checkpoint;
CUDA_VISIBLE_DEVICES=$GPU_ID python eval.py \
--train_dir="$MODEL_DIR" \
--model_checkpoint_path="${MODEL_DIR}/model.ckpt-${checkpoint}" \
--eval_data_pattern="/Youtube-8M/data/video/validate/validatea*" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--model=$MODEL \
--class_size=100 \
--moe_num_mixtures=4 \
--moe_layers=3 \
--run_once=True
fi
done
================================================
FILE: youtube-8m-zhangteng/eval_util.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Provides functions to help with evaluating models."""
import datetime
import numpy
from tensorflow.python.platform import gfile
import mean_average_precision_calculator as map_calculator
import average_precision_calculator as ap_calculator
def flatten(l):
""" Merges a list of lists into a single list. """
return [item for sublist in l for item in sublist]
def calculate_hit_at_one(predictions, actuals):
"""Performs a local (numpy) calculation of the hit at one.
Args:
predictions: Matrix containing the outputs of the model.
Dimensions are 'batch' x 'num_classes'.
actuals: Matrix containing the ground truth labels.
Dimensions are 'batch' x 'num_classes'.
Returns:
float: The average hit at one across the entire batch.
"""
top_prediction = numpy.argmax(predictions, 1)
hits = actuals[numpy.arange(actuals.shape[0]), top_prediction]
return numpy.average(hits)
def calculate_precision_at_equal_recall_rate(predictions, actuals):
"""Performs a local (numpy) calculation of the PERR.
Args:
predictions: Matrix containing the outputs of the model.
Dimensions are 'batch' x 'num_classes'.
actuals: Matrix containing the ground truth labels.
Dimensions are 'batch' x 'num_classes'.
Returns:
float: The average precision at equal recall rate across the entire batch.
"""
aggregated_precision = 0.0
num_videos = actuals.shape[0]
for row in numpy.arange(num_videos):
num_labels = int(numpy.sum(actuals[row]))
top_indices = numpy.argpartition(predictions[row],
-num_labels)[-num_labels:]
item_precision = 0.0
for label_index in top_indices:
if predictions[row][label_index] > 0:
item_precision += actuals[row][label_index]
item_precision /= top_indices.size
aggregated_precision += item_precision
aggregated_precision /= num_videos
return aggregated_precision
def calculate_gap(predictions, actuals, top_k=20):
"""Performs a local (numpy) calculation of the global average precision.
Only the top_k predictions are taken for each of the videos.
Args:
predictions: Matrix containing the outputs of the model.
Dimensions are 'batch' x 'num_classes'.
actuals: Matrix containing the ground truth labels.
Dimensions are 'batch' x 'num_classes'.
top_k: How many predictions to use per video.
Returns:
float: The global average precision.
"""
gap_calculator = ap_calculator.AveragePrecisionCalculator()
sparse_predictions, sparse_labels, num_positives = top_k_by_class(predictions, actuals, top_k)
gap_calculator.accumulate(flatten(sparse_predictions), flatten(sparse_labels), sum(num_positives))
return gap_calculator.peek_ap_at_n()
def top_k_by_class(predictions, labels, k=20):
"""Extracts the top k predictions for each video, sorted by class.
Args:
predictions: A numpy matrix containing the outputs of the model.
Dimensions are 'batch' x 'num_classes'.
k: the top k non-zero entries to preserve in each prediction.
Returns:
A tuple (predictions,labels, true_positives). 'predictions' and 'labels'
are lists of lists of floats. 'true_positives' is a list of scalars. The
length of the lists are equal to the number of classes. The entries in the
predictions variable are probability predictions, and
the corresponding entries in the labels variable are the ground truth for
those predictions. The entries in 'true_positives' are the number of true
positives for each class in the ground truth.
Raises:
ValueError: An error occurred when the k is not a positive integer.
"""
if k <= 0:
raise ValueError("k must be a positive integer.")
k = min(k, predictions.shape[1])
num_classes = predictions.shape[1]
prediction_triplets= []
for video_index in range(predictions.shape[0]):
prediction_triplets.extend(top_k_triplets(predictions[video_index],labels[video_index], k))
out_predictions = [[] for v in range(num_classes)]
out_labels = [[] for v in range(num_classes)]
for triplet in prediction_triplets:
out_predictions[triplet[0]].append(triplet[1])
out_labels[triplet[0]].append(triplet[2])
out_true_positives = [numpy.sum(labels[:,i]) for i in range(num_classes)]
return out_predictions, out_labels, out_true_positives
def top_k_triplets(predictions, labels, k=20):
"""Get the top_k for a 1-d numpy array. Returns a sparse list of tuples in
(prediction, class) format"""
m = len(predictions)
k = min(k, m)
indices = numpy.argpartition(predictions, -k)[-k:]
return [(index, predictions[index], labels[index]) for index in indices]
class EvaluationMetrics(object):
"""A class to store the evaluation metrics."""
def __init__(self, num_class, top_k):
"""Construct an EvaluationMetrics object to store the evaluation metrics.
Args:
num_class: A positive integer specifying the number of classes.
top_k: A positive integer specifying how many predictions are considered per video.
Raises:
ValueError: An error occurred when MeanAveragePrecisionCalculator cannot
not be constructed.
"""
self.sum_hit_at_one = 0.0
self.sum_perr = 0.0
self.sum_loss = 0.0
self.map_calculator = map_calculator.MeanAveragePrecisionCalculator(num_class)
self.global_ap_calculator = ap_calculator.AveragePrecisionCalculator()
self.top_k = top_k
self.num_examples = 0
def accumulate(self, predictions, labels, loss):
"""Accumulate the metrics calculated locally for this mini-batch.
Args:
predictions: A numpy matrix containing the outputs of the model.
Dimensions are 'batch' x 'num_classes'.
labels: A numpy matrix containing the ground truth labels.
Dimensions are 'batch' x 'num_classes'.
loss: A numpy array containing the loss for each sample.
Returns:
dictionary: A dictionary storing the metrics for the mini-batch.
Raises:
ValueError: An error occurred when the shape of predictions and actuals
does not match.
"""
batch_size = labels.shape[0]
mean_hit_at_one = calculate_hit_at_one(predictions, labels)
mean_perr = calculate_precision_at_equal_recall_rate(predictions, labels)
mean_loss = numpy.mean(loss)
# Take the top 20 predictions.
sparse_predictions, sparse_labels, num_positives = top_k_by_class(predictions, labels, self.top_k)
self.map_calculator.accumulate(sparse_predictions, sparse_labels, num_positives)
self.global_ap_calculator.accumulate(flatten(sparse_predictions), flatten(sparse_labels), sum(num_positives))
self.num_examples += batch_size
self.sum_hit_at_one += mean_hit_at_one * batch_size
self.sum_perr += mean_perr * batch_size
self.sum_loss += mean_loss * batch_size
return {"hit_at_one": mean_hit_at_one, "perr": mean_perr, "loss": mean_loss}
def get(self):
"""Calculate the evaluation metrics for the whole epoch.
Raises:
ValueError: If no examples were accumulated.
Returns:
dictionary: a dictionary storing the evaluation metrics for the epoch. The
dictionary has the fields: avg_hit_at_one, avg_perr, avg_loss, and
aps (default nan).
"""
if self.num_examples <= 0:
raise ValueError("total_sample must be positive.")
avg_hit_at_one = self.sum_hit_at_one / self.num_examples
avg_perr = self.sum_perr / self.num_examples
avg_loss = self.sum_loss / self.num_examples
aps = self.map_calculator.peek_map_at_n()
gap = self.global_ap_calculator.peek_ap_at_n()
epoch_info_dict = {}
return {"avg_hit_at_one": avg_hit_at_one, "avg_perr": avg_perr,
"avg_loss": avg_loss, "aps": aps, "gap": gap}
def clear(self):
"""Clear the evaluation metrics and reset the EvaluationMetrics object."""
self.sum_hit_at_one = 0.0
self.sum_perr = 0.0
self.sum_loss = 0.0
self.map_calculator.clear()
self.global_ap_calculator.clear()
self.num_examples = 0
================================================
FILE: youtube-8m-zhangteng/frame_level_models.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains a collection of models which operate on variable-length sequences.
"""
import math
import models
import video_level_models
import tensorflow as tf
import model_utils as utils
import numpy as np
import tensorflow.contrib.slim as slim
from tensorflow.python.ops import tensor_array_ops, control_flow_ops
from tensorflow import flags
import rnn_residual
FLAGS = flags.FLAGS
flags.DEFINE_integer("iterations", 30,
"Number of frames per batch for DBoF.")
flags.DEFINE_bool("dbof_add_batch_norm", True,
"Adds batch normalization to the DBoF model.")
flags.DEFINE_bool(
"sample_random_frames", True,
"If true samples random frames (for frame level models). If false, a random"
"sequence of frames is sampled instead.")
flags.DEFINE_integer("dbof_cluster_size", 8192,
"Number of units in the DBoF cluster layer.")
flags.DEFINE_integer("dbof_hidden_size", 1024,
"Number of units in the DBoF hidden layer.")
flags.DEFINE_string("dbof_pooling_method", "max",
"The pooling method used in the DBoF cluster layer. "
"Choices are 'average' and 'max'.")
flags.DEFINE_string("video_level_classifier_model", "MoeModel",
"Some Frame-Level models can be decomposed into a "
"generalized pooling operation followed by a "
"classifier layer")
flags.DEFINE_integer("lstm_cells", 1024, "Number of LSTM cells.")
flags.DEFINE_integer("lstm_length", 10, "Number of LSTM length, only used in LstmLayerModel.")
flags.DEFINE_integer("lstm_layers", 2, "Number of LSTM layers.")
flags.DEFINE_integer("lstm_interval", 3, "Number of LSTM residual intervals, only used in LstmResidualModel.")
flags.DEFINE_bool("train", True,
"Whether the process is training procedure."
"used for batch normalization and LstmRandomModel and LstmNioseModel.")
flags.DEFINE_integer("cnn_cells", 256, "Number of CNN cells.")
class FrameLevelLogisticModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a logistic classifier over the average of the
frame-level features.
This class is intended to be an example for implementors of frame level
models. If you want to train a model over averaged features it is more
efficient to average them beforehand rather than on the fly.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
num_frames = tf.cast(tf.expand_dims(num_frames, 1), tf.float32)
feature_size = model_input.get_shape().as_list()[2]
max_frames = model_input.get_shape().as_list()[1]
denominators = tf.reshape(
tf.tile(num_frames, [1, feature_size]), [-1, feature_size])
avg_pooled = tf.reduce_sum(model_input,
axis=[1]) / denominators
output = slim.fully_connected(
avg_pooled, vocab_size, activation_fn=tf.nn.sigmoid,
weights_regularizer=slim.l2_regularizer(1e-8))
return {"predictions": output}
class DbofModel(models.BaseModel):
"""Creates a Deep Bag of Frames model.
The model projects the features for each frame into a higher dimensional
'clustering' space, pools across frames in that space, and then
uses a configurable video-level model to classify the now aggregated features.
The model will randomly sample either frames or sequences of frames during
training to speed up convergence.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
def create_model(self,
model_input,
vocab_size,
num_frames,
iterations=None,
add_batch_norm=None,
sample_random_frames=None,
cluster_size=None,
hidden_size=None,
is_training=True,
**unused_params):
iterations = iterations or FLAGS.iterations
add_batch_norm = add_batch_norm or FLAGS.dbof_add_batch_norm
random_frames = sample_random_frames or FLAGS.sample_random_frames
cluster_size = cluster_size or FLAGS.dbof_cluster_size
hidden1_size = hidden_size or FLAGS.dbof_hidden_size
num_frames = tf.cast(tf.expand_dims(num_frames, 1), tf.float32)
if random_frames:
model_input = utils.SampleRandomFrames(model_input, num_frames,
iterations)
else:
model_input = utils.SampleRandomSequence(model_input, num_frames,
iterations)
max_frames = model_input.get_shape().as_list()[1]
feature_size = model_input.get_shape().as_list()[2]
reshaped_input = tf.reshape(model_input, [-1, feature_size])
tf.summary.histogram("input_hist", reshaped_input)
if add_batch_norm:
reshaped_input = slim.batch_norm(
reshaped_input,
center=True,
scale=True,
is_training=is_training,
scope="input_bn")
cluster_weights = tf.Variable(tf.random_normal(
[feature_size, cluster_size],
stddev=1 / math.sqrt(feature_size)))
tf.summary.histogram("cluster_weights", cluster_weights)
activation = tf.matmul(reshaped_input, cluster_weights)
if add_batch_norm:
activation = slim.batch_norm(
activation,
center=True,
scale=True,
is_training=is_training,
scope="cluster_bn")
else:
cluster_biases = tf.Variable(
tf.random_normal(
[cluster_size], stddev=1 / math.sqrt(feature_size)))
tf.summary.histogram("cluster_biases", cluster_biases)
activation += cluster_biases
activation = tf.nn.relu6(activation)
tf.summary.histogram("cluster_output", activation)
activation = tf.reshape(activation, [-1, max_frames, cluster_size])
activation = utils.FramePooling(activation, FLAGS.dbof_pooling_method)
hidden1_weights = tf.Variable(tf.random_normal(
[cluster_size, hidden1_size],
stddev=1 / math.sqrt(cluster_size)))
tf.summary.histogram("hidden1_weights", hidden1_weights)
activation = tf.matmul(activation, hidden1_weights)
if add_batch_norm:
activation = slim.batch_norm(
activation,
center=True,
scale=True,
is_training=is_training,
scope="hidden1_bn")
else:
hidden1_biases = tf.Variable(
tf.random_normal(
[hidden1_size], stddev=0.01))
tf.summary.histogram("hidden1_biases", hidden1_biases)
activation += hidden1_biases
activation = tf.nn.relu6(activation)
tf.summary.histogram("hidden1_output", activation)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=activation,
vocab_size=vocab_size,
**unused_params)
class Dbof3mModel(models.BaseModel):
"""Creates a Deep Bag of Frames model.
The model projects the features for each frame into a higher dimensional
'clustering' space, pools across frames in that space, and then
uses a configurable video-level model to classify the now aggregated features.
The model will randomly sample either frames or sequences of frames during
training to speed up convergence.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
def sub_moe(self,
model_input,
vocab_size,
num_mixtures = None,
l2_penalty=1e-8,
scopename="",
**unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+scopename)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts"+scopename)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return model_input, final_probabilities
def create_model(self,
model_input,
vocab_size,
num_frames,
**unused_params):
shape = model_input.get_shape().as_list()
frames_sum = tf.reduce_sum(tf.abs(model_input),axis=2)
frames_true = tf.ones(tf.shape(frames_sum))
frames_false = tf.zeros(tf.shape(frames_sum))
frames_bool = tf.reshape(tf.where(tf.greater(frames_sum, frames_false), frames_true, frames_false),[-1,shape[1],1])
activation_1 = tf.reduce_max(model_input, axis=1)
activation_2 = tf.reduce_sum(model_input*frames_bool, axis=1)/(tf.reduce_sum(frames_bool, axis=1)+1e-6)
activation_3 = tf.reduce_min(model_input, axis=1)
model_input_1, final_probilities_1 = self.sub_moe(activation_1,vocab_size,scopename="_max")
model_input_2, final_probilities_2 = self.sub_moe(activation_2,vocab_size,scopename="_mean")
model_input_3, final_probilities_3 = self.sub_moe(activation_3,vocab_size,scopename="_min")
final_probilities = tf.stack((final_probilities_1,final_probilities_2,final_probilities_3),axis=1)
weight2d = tf.get_variable("ensemble_weight2d",
shape=[shape[2], 3, vocab_size],
regularizer=slim.l2_regularizer(1.0e-8))
activations = tf.stack((model_input_1, model_input_2, model_input_3), axis=2)
weight = tf.nn.softmax(tf.einsum("aij,ijk->ajk", activations, weight2d), dim=1)
result = {}
result["prediction_frames"] = tf.reshape(final_probilities,[-1,vocab_size])
result["predictions"] = tf.reduce_sum(final_probilities*weight,axis=1)
return result
class batch_norm(object):
"""Code modification of http://stackoverflow.com/a/33950177"""
def __init__(self, epsilon=1e-5, momentum=0.99, name="batch_norm"):
with tf.variable_scope(name):
self.epsilon = epsilon
self.momentum = momentum
self.ema = tf.train.ExponentialMovingAverage(decay=self.momentum)
self.name = name
def __call__(self, x, train=True):
shape = x.get_shape().as_list()
with tf.variable_scope(self.name) as scope:
self.beta = tf.get_variable("beta", shape[1:],
initializer=tf.constant_initializer(0.))
self.gamma = tf.get_variable("gamma", shape[1:],
initializer=tf.random_normal_initializer(1.,0.02))
self.mean = tf.get_variable("mean", shape[1:],
initializer=tf.constant_initializer(0.),trainable=False)
self.variance = tf.get_variable("variance",shape[1:],
initializer=tf.constant_initializer(1.),trainable=False)
if train:
batch_mean, batch_var = tf.nn.moments(x, [0], name='moments')
self.mean.assign(batch_mean)
self.variance.assign(batch_var)
ema_apply_op = self.ema.apply([self.mean, self.variance])
with tf.control_dependencies([ema_apply_op]):
mean, var = tf.identity(batch_mean), tf.identity(batch_var)
else:
mean, var = self.ema.average(self.mean), self.ema.average(self.variance)
normed = tf.nn.batch_normalization(x, mean, var, self.beta, self.gamma, self.epsilon)
return normed
class LstmVisionModel(models.BaseModel):
"""Logistic model with L2 regularization."""
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
"""Creates a matrix regression model.
Args:
model_input: 'batch' x 'num_features' x 'num_methods' matrix of input features.
vocab_size: The number of classes in the dataset.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes."""
lstm_size = FLAGS.lstm_cells
max_frames = model_input.get_shape().as_list()[1]
emb_dim = model_input.get_shape().as_list()[2]
hidden_dim = lstm_size
with tf.variable_scope('lstm_forward'):
g_recurrent_unit_forward = self.create_recurrent_unit(emb_dim,hidden_dim,l2_penalty)
h0 = tf.zeros([tf.shape(model_input)[0], hidden_dim])
h0 = tf.stack([h0, h0])
outputs_gate = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
outputs_state = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
def _pretrain_forward(i,h_tm1,g_predictions,s_predictions):
x_t = model_input[:,i,:]
gate, h_t = g_recurrent_unit_forward(x_t, h_tm1)
hidden_state, c_prev = tf.unstack(h_t)
g_predictions = g_predictions.write(i,gate)
s_predictions = s_predictions.write(i,c_prev)
return i + 1, h_t, g_predictions, s_predictions
_, _, gate_outputs, state_outputs = control_flow_ops.while_loop(
cond=lambda i, _1, _2, _3: i < max_frames,
body=_pretrain_forward,
loop_vars=(tf.constant(0, dtype=tf.int32),h0,outputs_gate,outputs_state))
gate_outputs = gate_outputs.stack()
state_outputs = state_outputs.stack()
batch_size = tf.shape(model_input)[0]
index_1 = tf.range(0, batch_size) * max_frames + (num_frames - 1)
gate_outputs = tf.transpose(gate_outputs, [1, 0, 2])
gate_outputs = tf.gather(tf.reshape(gate_outputs, [-1, hidden_dim]), index_1)
state_outputs = tf.transpose(state_outputs, [1, 0, 2])
state_outputs = tf.gather(tf.reshape(state_outputs, [-1, hidden_dim]), index_1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=state_outputs,
vocab_size=vocab_size,
**unused_params)
def create_recurrent_unit(self,emb_dim,hidden_dim,l2_penalty):
# Weights and Bias for input and hidden tensor
Wi = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wi))
Ui = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Ui")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Ui))
bi = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bi))
Wf = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wf))
Uf = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uf))
bf = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bf))
Wog = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wog))
Uog = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uog))
bog = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bog))
Wc = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wc))
Uc = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uc))
bc = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bc))
bn0 = batch_norm(name='bn0')
bn1 = batch_norm(name='bn1')
bn2 = batch_norm(name='bn2')
bn3 = batch_norm(name='bn3')
gn0 = batch_norm(name='gn0')
gn1 = batch_norm(name='gn1')
gn2 = batch_norm(name='gn2')
gn3 = batch_norm(name='gn3')
def unit(x, hidden_memory_tm1):
previous_hidden_state, c_prev = tf.unstack(hidden_memory_tm1)
# Input Gate
i = tf.sigmoid(
bn0(tf.matmul(x, Wi), train=FLAGS.train) +
gn0(tf.matmul(previous_hidden_state, Ui), train=FLAGS.train) + bi
)
# Forget Gate
f = tf.sigmoid(
bn1(tf.matmul(x, Wf), train=FLAGS.train) +
gn1(tf.matmul(previous_hidden_state, Uf), train=FLAGS.train) + bf
)
# Output Gate
o = tf.sigmoid(
bn2(tf.matmul(x, Wog), train=FLAGS.train) +
gn2(tf.matmul(previous_hidden_state, Uog), train=FLAGS.train) + bog
)
# New Memory Cell
c_ = tf.nn.tanh(
bn3(tf.matmul(x, Wc), train=FLAGS.train) +
gn3(tf.matmul(previous_hidden_state, Uc), train=FLAGS.train) + bc
)
# Final Memory cell
c = f * c_prev + i * c_
# Current Hidden state
current_hidden_state = o * tf.nn.tanh(c)
return f, tf.stack([current_hidden_state, c])
return unit
class LstmGluModel(models.BaseModel):
"""Logistic model with L2 regularization."""
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
"""Creates a matrix regression model.
Args:
model_input: 'batch' x 'num_features' x 'num_methods' matrix of input features.
vocab_size: The number of classes in the dataset.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes."""
lstm_size = FLAGS.lstm_cells
max_frames = model_input.get_shape().as_list()[1]
emb_dim = model_input.get_shape().as_list()[2]
hidden_dim = lstm_size
with tf.variable_scope('lstm_forward'):
g_recurrent_unit_forward = self.create_recurrent_unit(emb_dim,hidden_dim,l2_penalty)
h0 = tf.zeros([tf.shape(model_input)[0], hidden_dim])
h1 = tf.zeros([tf.shape(model_input)[0], emb_dim])
h0 = tf.stack([h0, h0])
h1 = tf.stack([h1, h1])
outputs_gate = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
outputs_state = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
def _pretrain_forward(i, h_tm0, h_tm1, g_predictions, s_predictions):
x_t = model_input[:,i,:]
gate, h_t0, h_t1 = g_recurrent_unit_forward(x_t, h_tm0, h_tm1)
hidden_state, c_prev = tf.unstack(h_t1)
g_predictions = g_predictions.write(i,gate)
s_predictions = s_predictions.write(i,c_prev)
return i + 1, h_t0, h_t1, g_predictions, s_predictions
_, _, _, gate_outputs, state_outputs = control_flow_ops.while_loop(
cond=lambda i, _1, _2, _3, _4: i < max_frames,
body=_pretrain_forward,
loop_vars=(tf.constant(0, dtype=tf.int32), h1, h0,outputs_gate,outputs_state))
gate_outputs = gate_outputs.stack()
state_outputs = state_outputs.stack()
batch_size = tf.shape(model_input)[0]
index_1 = tf.range(0, batch_size) * max_frames + (num_frames - 1)
gate_outputs = tf.transpose(gate_outputs, [1, 0, 2])
gate_outputs = tf.gather(tf.reshape(gate_outputs, [-1, hidden_dim]), index_1)
state_outputs = tf.transpose(state_outputs, [1, 0, 2])
state_outputs = tf.gather(tf.reshape(state_outputs, [-1, hidden_dim]), index_1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=state_outputs,
vocab_size=vocab_size,
**unused_params)
def create_recurrent_unit(self,emb_dim,hidden_dim,l2_penalty):
# Weights and Bias for input and hidden tensor
Wi = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wi))
Ui = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Ui")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Ui))
Vi = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Vi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vi))
bi = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bi))
Wf = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wf))
Vf = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Vf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vf))
Uf = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uf))
bf = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bf))
Wog = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wog))
Vog = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Vog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vog))
Uog = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uog))
bog = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bog))
Wc = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wc))
Vc = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Vc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vc))
Uc = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uc))
bc = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bc))
Wix = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wix")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wix))
Vix = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Vix")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vix))
Uix = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Uix")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uix))
bix = tf.Variable(tf.constant(0.1, shape=[1]), name="bix")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bix))
Wfx = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wfx")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wfx))
Vfx = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Vfx")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vfx))
Ufx = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Ufx")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Ufx))
bfx = tf.Variable(tf.constant(0.1, shape=[1]), name="bfx")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bfx))
def unit(x, hidden_memory_tm0, hidden_memory_tm1):
previous_hidden_x, x_prev = tf.unstack(hidden_memory_tm0)
previous_hidden_state, c_prev = tf.unstack(hidden_memory_tm1)
# Input Gate
i = tf.sigmoid(
tf.matmul(x, Wi) + tf.matmul(previous_hidden_x, Vi) +
tf.matmul(previous_hidden_state, Ui) + bi
)
# Forget Gate
f = tf.sigmoid(
tf.matmul(x, Wf) + tf.matmul(previous_hidden_x, Vf) +
tf.matmul(previous_hidden_state, Uf) + bf
)
# Input Gate
ix = tf.sigmoid(
tf.matmul(x, Wix) + tf.matmul(previous_hidden_x, Vix) +
tf.matmul(previous_hidden_state, Uix) + bix
)
# Forget Gate
fx = tf.sigmoid(
tf.matmul(x, Wfx) + tf.matmul(previous_hidden_x, Vfx) +
tf.matmul(previous_hidden_state, Ufx) + bfx
)
# Output Gate
o = tf.sigmoid(
tf.matmul(x, Wog) + tf.matmul(previous_hidden_x, Vog) +
tf.matmul(previous_hidden_state, Uog) + bog
)
# New Memory Cell
c_ = tf.tanh(
tf.matmul(x, Wc) + tf.matmul(previous_hidden_x, Vc) +
tf.matmul(previous_hidden_state, Uc) + bc
)
# Final Memory cell
c = f * c_prev + i * c_
current_x = fx * x_prev + ix * x
current_hidden_x = tf.nn.l2_normalize(current_x, dim=1)
# Current Hidden state
current_hidden_state = o * tf.tanh(c)
return f, tf.stack([current_hidden_x, current_x]), tf.stack([current_hidden_state, c])
return unit
class LstmGlu2Model(models.BaseModel):
"""Logistic model with L2 regularization."""
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
"""Creates a matrix regression model.
Args:
model_input: 'batch' x 'num_features' x 'num_methods' matrix of input features.
vocab_size: The number of classes in the dataset.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes."""
lstm_size = FLAGS.lstm_cells
max_frames = model_input.get_shape().as_list()[1]
emb_dim = model_input.get_shape().as_list()[2]
hidden_dim = lstm_size
with tf.variable_scope('lstm_forward'):
g_recurrent_unit_forward = self.create_recurrent_unit(emb_dim,hidden_dim,l2_penalty)
h0 = tf.zeros([tf.shape(model_input)[0], hidden_dim])
h1 = tf.zeros([tf.shape(model_input)[0], emb_dim])
h0 = tf.stack([h0, h0])
h1 = tf.stack([h1, h1])
outputs_gate = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
outputs_state = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
def _pretrain_forward(i, h_tm0, h_tm1, g_predictions, s_predictions):
x_t = model_input[:,i,:]
gate, h_t0, h_t1 = g_recurrent_unit_forward(x_t, h_tm0, h_tm1)
hidden_state, c_prev = tf.unstack(h_t1)
g_predictions = g_predictions.write(i,gate)
s_predictions = s_predictions.write(i,c_prev)
return i + 1, h_t0, h_t1, g_predictions, s_predictions
_, _, _, gate_outputs, state_outputs = control_flow_ops.while_loop(
cond=lambda i, _1, _2, _3, _4: i < max_frames,
body=_pretrain_forward,
loop_vars=(tf.constant(0, dtype=tf.int32), h1, h0,outputs_gate,outputs_state))
gate_outputs = gate_outputs.stack()
state_outputs = state_outputs.stack()
batch_size = tf.shape(model_input)[0]
index_1 = tf.range(0, batch_size) * max_frames + (num_frames - 1)
gate_outputs = tf.transpose(gate_outputs, [1, 0, 2])
gate_outputs = tf.gather(tf.reshape(gate_outputs, [-1, hidden_dim]), index_1)
state_outputs = tf.transpose(state_outputs, [1, 0, 2])
state_outputs = tf.gather(tf.reshape(state_outputs, [-1, hidden_dim]), index_1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=state_outputs,
vocab_size=vocab_size,
**unused_params)
def create_recurrent_unit(self,emb_dim,hidden_dim,l2_penalty):
# Weights and Bias for input and hidden tensor
Wi = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wi))
Ui = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Ui")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Ui))
Vi = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Vi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vi))
bi = tf.Variable(tf.constant(0.1, shape=[1]), name="bi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bi))
Wf = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wf))
Vf = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Vf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vf))
Uf = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Uf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uf))
bf = tf.Variable(tf.constant(1.0, shape=[1]), name="bf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bf))
Wog = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wog))
Vog = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Vog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vog))
Uog = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uog))
bog = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bog))
Wc = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wc))
Vc = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Vc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vc))
Uc = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uc))
bc = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bc))
Wix = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wix")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wix))
Vix = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Vix")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vix))
Uix = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Uix")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uix))
bix = tf.Variable(tf.constant(0.1, shape=[1]), name="bix")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bix))
Wfx = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wfx")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wfx))
Vfx = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Vfx")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vfx))
Ufx = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Ufx")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Ufx))
bfx = tf.Variable(tf.constant(1.0, shape=[1]), name="bfx")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bfx))
def unit(x, hidden_memory_tm0, hidden_memory_tm1):
previous_hidden_x, x_prev = tf.unstack(hidden_memory_tm0)
previous_hidden_state, c_prev = tf.unstack(hidden_memory_tm1)
# Input Gate
i = tf.sigmoid(
tf.matmul(x, Wi) + tf.matmul(previous_hidden_x, Vi) +
tf.matmul(previous_hidden_state, Ui) + bi
)
# Forget Gate
f = tf.sigmoid(
tf.matmul(x, Wf) + tf.matmul(previous_hidden_x, Vf) +
tf.matmul(previous_hidden_state, Uf) + bf
)
# Input Gate
ix = tf.sigmoid(
tf.matmul(x, Wix) + tf.matmul(previous_hidden_x, Vix) +
tf.matmul(previous_hidden_state, Uix) + bix
)
# Forget Gate
fx = tf.sigmoid(
tf.matmul(x, Wfx) + tf.matmul(previous_hidden_x, Vfx) +
tf.matmul(previous_hidden_state, Ufx) + bfx
)
# Output Gate
o = tf.sigmoid(
tf.matmul(x, Wog) + tf.matmul(previous_hidden_x, Vog) +
tf.matmul(previous_hidden_state, Uog) + bog
)
# New Memory Cell
c_ = tf.tanh(
tf.matmul(x, Wc) + tf.matmul(previous_hidden_x, Vc) +
tf.matmul(previous_hidden_state, Uc) + bc
)
# Final Memory cell
c = f * c_prev + i * c_
current_x = fx * x_prev + ix * x
current_hidden_x = tf.nn.l2_normalize(current_x, dim=1)
# Current Hidden state
current_hidden_state = o * tf.tanh(c)
return f, tf.stack([current_hidden_x, current_x]), tf.stack([current_hidden_state, c])
return unit
class LstmGlu2MultilayerModel(models.BaseModel):
"""Logistic model with L2 regularization."""
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
"""Creates a matrix regression model.
Args:
model_input: 'batch' x 'num_features' x 'num_methods' matrix of input features.
vocab_size: The number of classes in the dataset.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes."""
lstm_size = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
hidden_outputs = model_input
state_outputs = []
for i in range(number_of_layers):
state_output, hidden_outputs = self.rnn_gate(hidden_outputs, lstm_size, num_frames, sub_scope="lstm_lsyer%d" % i)
state_outputs.append(state_output)
state_outputs = tf.concat(state_outputs,axis=1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=state_outputs,
vocab_size=vocab_size,
**unused_params)
def rnn_gate(self, model_input, lstm_size, num_frames, l2_penalty=1e-8, sub_scope="", **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
## Batch normalize the input
max_frames = model_input.get_shape().as_list()[1]
emb_dim = model_input.get_shape().as_list()[2]
hidden_dim = lstm_size
with tf.variable_scope(sub_scope+'lstm_forward'):
g_recurrent_unit_forward = LstmGlu2Model().create_recurrent_unit(emb_dim,hidden_dim,l2_penalty)
h0 = tf.zeros([tf.shape(model_input)[0], hidden_dim])
h1 = tf.zeros([tf.shape(model_input)[0], emb_dim])
h0 = tf.stack([h0, h0])
h1 = tf.stack([h1, h1])
outputs_hidden = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
outputs_state = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
def _pretrain_forward(i, h_tm0, h_tm1, g_predictions, s_predictions):
x_t = model_input[:,i,:]
gate, h_t0, h_t1 = g_recurrent_unit_forward(x_t, h_tm0, h_tm1)
hidden_state, c_prev = tf.unstack(h_t1)
g_predictions = g_predictions.write(i,hidden_state)
s_predictions = s_predictions.write(i,c_prev)
return i + 1, h_t0, h_t1, g_predictions, s_predictions
_, _, _, hidden_outputs, state_outputs = control_flow_ops.while_loop(
cond=lambda i, _1, _2, _3, _4: i < max_frames,
body=_pretrain_forward,
loop_vars=(tf.constant(0, dtype=tf.int32), h1, h0,outputs_hidden,outputs_state))
state_outputs = state_outputs.stack()
state_outputs = tf.transpose(state_outputs, [1, 0, 2])
hidden_outputs = hidden_outputs.stack()
hidden_outputs = tf.transpose(hidden_outputs, [1, 0, 2])
hidden_outputs = tf.reshape(hidden_outputs, [-1, max_frames, lstm_size])
batch_size = tf.shape(model_input)[0]
index_1 = tf.range(0, batch_size) * max_frames + (num_frames - 1)
state_outputs = tf.gather(tf.reshape(state_outputs, [-1, hidden_dim]), index_1)
return state_outputs, hidden_outputs
class LstmBigluModel(models.BaseModel):
"""Logistic model with L2 regularization."""
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
"""Creates a matrix regression model.
Args:
model_input: 'batch' x 'num_features' x 'num_methods' matrix of input features.
vocab_size: The number of classes in the dataset.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes."""
lstm_size = FLAGS.lstm_cells
max_frames = model_input.get_shape().as_list()[1]
emb_dim = model_input.get_shape().as_list()[2]
hidden_dim = lstm_size
model_input_reverse = tf.reverse_sequence(model_input,num_frames,seq_axis=1)
with tf.variable_scope('lstm_forward'):
g_recurrent_unit_forward = self.create_forward_recurrent_unit(emb_dim,hidden_dim,l2_penalty)
with tf.variable_scope('lstm_backward'):
g_recurrent_unit_backward = self.create_backward_recurrent_unit(emb_dim,hidden_dim,l2_penalty)
h0 = tf.zeros([tf.shape(model_input)[0], hidden_dim])
h0 = tf.stack([h0, h0])
forward_outputs_x = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
forward_outputs_state = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
def _pretrain_forward(i, h_tm1, g_predictions, s_predictions):
x_t = model_input_reverse[:,i,:]
_, h_t1 = g_recurrent_unit_forward(x_t, h_tm1)
hidden_state, c_prev = tf.unstack(h_t1)
g_predictions = g_predictions.write(i, hidden_state)
s_predictions = s_predictions.write(i,c_prev)
return i + 1, h_t1, g_predictions, s_predictions
_, _, forward_x_outputs, forward_state_outputs = control_flow_ops.while_loop(
cond=lambda i, _1, _2, _3: i < max_frames,
body=_pretrain_forward,
loop_vars=(tf.constant(0, dtype=tf.int32), h0, forward_outputs_x, forward_outputs_state))
forward_x_outputs = forward_x_outputs.stack()
forward_state_outputs = forward_state_outputs.stack()
batch_size = tf.shape(model_input)[0]
index_1 = tf.range(0, batch_size) * max_frames + (num_frames - 1)
forward_x_outputs = tf.transpose(forward_x_outputs, [1, 0, 2])
x_outputs_reverse = tf.reverse_sequence(forward_x_outputs,num_frames,seq_axis=1)
forward_state_outputs = tf.transpose(forward_state_outputs, [1, 0, 2])
forward_state_outputs = tf.gather(tf.reshape(forward_state_outputs, [-1, hidden_dim]), index_1)
h0 = tf.zeros([tf.shape(model_input)[0], hidden_dim])
h0 = tf.stack([h0, h0])
backward_outputs_state = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
def _pretrain_backward(i, h_tm1, s_predictions):
x_t = model_input[:,i,:]
y_t = x_outputs_reverse[:,i,:]
_, h_t1 = g_recurrent_unit_backward(x_t, y_t, h_tm1)
hidden_state, c_prev = tf.unstack(h_t1)
s_predictions = s_predictions.write(i,c_prev)
return i + 1, h_t1, s_predictions
_, _, backward_state_outputs = control_flow_ops.while_loop(
cond=lambda i, _1, _2: i < max_frames,
body=_pretrain_backward,
loop_vars=(tf.constant(0, dtype=tf.int32), h0, backward_outputs_state))
backward_state_outputs = backward_state_outputs.stack()
backward_state_outputs = tf.transpose(backward_state_outputs, [1, 0, 2])
backward_state_outputs = tf.gather(tf.reshape(backward_state_outputs, [-1, hidden_dim]), index_1)
moe_input = tf.concat((forward_state_outputs, backward_state_outputs), axis=1)
final_probabilities = self.sub_moe(moe_input, vocab_size, scopename="backward")
#probabilities_by_class = self.sub_moe(forward_state_outputs, vocab_size, scopename="forward")
return {"predictions": final_probabilities}
def create_forward_recurrent_unit(self,emb_dim,hidden_dim,l2_penalty):
# Weights and Bias for input and hidden tensor
Wi = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wi))
Ui = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Ui")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Ui))
bi = tf.Variable(tf.constant(0.1, shape=[1]), name="bi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bi))
Wf = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wf))
Uf = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Uf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uf))
bf = tf.Variable(tf.constant(0.1, shape=[1]), name="bf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bf))
Wog = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wog))
Uog = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uog))
bog = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bog))
Wc = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wc))
Uc = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uc))
bc = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bc))
def unit(x, hidden_memory_tm1):
previous_hidden_state, c_prev = tf.unstack(hidden_memory_tm1)
# Input Gate
i = tf.sigmoid(
tf.matmul(x, Wi) +
tf.matmul(previous_hidden_state, Ui) + bi
)
# Forget Gate
f = tf.sigmoid(
tf.matmul(x, Wf) +
tf.matmul(previous_hidden_state, Uf) + bf
)
# Output Gate
o = tf.sigmoid(
tf.matmul(x, Wog) +
tf.matmul(previous_hidden_state, Uog) + bog
)
# New Memory Cell
c_ = tf.nn.tanh(
tf.matmul(x, Wc)+
tf.matmul(previous_hidden_state, Uc) + bc
)
# Final Memory cell
c = f * c_prev + i * c_
# Current Hidden state
current_hidden_state = o * tf.nn.tanh(c)
return f, tf.stack([current_hidden_state, c])
return unit
def create_backward_recurrent_unit(self,emb_dim,hidden_dim,l2_penalty):
# Weights and Bias for input and hidden tensor
Wi = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wi))
Ui = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Ui")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Ui))
Vi = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Vi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vi))
bi = tf.Variable(tf.constant(0.1, shape=[1]), name="bi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bi))
Wf = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wf))
Vf = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Vf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vf))
Uf = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Uf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uf))
bf = tf.Variable(tf.constant(0.1, shape=[1]), name="bf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bf))
Wog = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wog))
Vog = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Vog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vog))
Uog = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uog))
bog = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bog))
Wc = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wc))
Vc = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Vc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vc))
Uc = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uc))
bc = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bc))
def unit(x, y, hidden_memory_tm1):
previous_hidden_state, c_prev = tf.unstack(hidden_memory_tm1)
# Input Gate
i = tf.sigmoid(
tf.matmul(x, Wi) + tf.matmul(y, Vi)+
tf.matmul(previous_hidden_state, Ui) + bi
)
# Forget Gate
f = tf.sigmoid(
tf.matmul(x, Wf) + tf.matmul(y, Vf) +
tf.matmul(previous_hidden_state, Uf) + bf
)
# Output Gate
o = tf.sigmoid(
tf.matmul(x, Wog) + tf.matmul(y, Vog) +
tf.matmul(previous_hidden_state, Uog) + bog
)
# New Memory Cell
c_ = tf.tanh(
tf.matmul(x, Wc) + tf.matmul(y, Vc) +
tf.matmul(previous_hidden_state, Uc) + bc
)
# Final Memory cell
c = f * c_prev + i * c_
# Current Hidden state
current_hidden_state = o * tf.tanh(c)
return f, tf.stack([current_hidden_state, c])
return unit
def sub_moe(self,model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
scopename="",
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+scopename)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts"+scopename)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
class LstmBiglu2Model(models.BaseModel):
"""Logistic model with L2 regularization."""
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
"""Creates a matrix regression model.
Args:
model_input: 'batch' x 'num_features' x 'num_methods' matrix of input features.
vocab_size: The number of classes in the dataset.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes."""
lstm_size = FLAGS.lstm_cells
max_frames = model_input.get_shape().as_list()[1]
emb_dim = model_input.get_shape().as_list()[2]
hidden_dim = lstm_size
model_input_reverse = tf.reverse_sequence(model_input,num_frames,seq_axis=1)
with tf.variable_scope('lstm_forward'):
g_recurrent_unit_forward = self.create_forward_recurrent_unit(emb_dim,hidden_dim,l2_penalty)
with tf.variable_scope('lstm_backward'):
g_recurrent_unit_backward = self.create_backward_recurrent_unit(emb_dim,hidden_dim,l2_penalty)
h0 = tf.zeros([tf.shape(model_input)[0], hidden_dim])
h1 = tf.zeros([tf.shape(model_input)[0], emb_dim])
h0 = tf.stack([h0, h0])
h1 = tf.stack([h1, h1])
forward_outputs_x = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
forward_outputs_state = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
def _pretrain_forward(i, h_tm0, h_tm1, g_predictions, s_predictions):
x_t = model_input_reverse[:,i,:]
gate, h_t0, h_t1 = g_recurrent_unit_forward(x_t, h_tm0, h_tm1)
hidden_x, x_prev = tf.unstack(h_t0)
hidden_state, c_prev = tf.unstack(h_t1)
g_predictions = g_predictions.write(i,hidden_x)
s_predictions = s_predictions.write(i,c_prev)
return i + 1, h_t0, h_t1, g_predictions, s_predictions
_, _, _, forward_x_outputs, forward_state_outputs = control_flow_ops.while_loop(
cond=lambda i, _1, _2, _3, _4: i < max_frames,
body=_pretrain_forward,
loop_vars=(tf.constant(0, dtype=tf.int32), h1, h0,forward_outputs_x,forward_outputs_state))
forward_x_outputs = forward_x_outputs.stack()
forward_state_outputs = forward_state_outputs.stack()
batch_size = tf.shape(model_input)[0]
index_1 = tf.range(0, batch_size) * max_frames + (num_frames - 1)
forward_x_outputs = tf.transpose(forward_x_outputs, [1, 0, 2])
x_outputs_reverse = tf.reverse_sequence(forward_x_outputs,num_frames,seq_axis=1)
forward_state_outputs = tf.transpose(forward_state_outputs, [1, 0, 2])
forward_state_outputs = tf.gather(tf.reshape(forward_state_outputs, [-1, hidden_dim]), index_1)
h0 = tf.zeros([tf.shape(model_input)[0], hidden_dim])
h1 = tf.zeros([tf.shape(model_input)[0], emb_dim])
h0 = tf.stack([h0, h0])
h1 = tf.stack([h1, h1])
backward_outputs_state = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
def _pretrain_backward(i, h_tm0, h_tm1, s_predictions):
x_t = model_input[:,i,:]
y_t = x_outputs_reverse[:,i,:]
gate, h_t0, h_t1 = g_recurrent_unit_backward(x_t, y_t, h_tm0, h_tm1)
hidden_state, c_prev = tf.unstack(h_t1)
s_predictions = s_predictions.write(i,c_prev)
return i + 1, h_t0, h_t1, s_predictions
_, _, _, backward_state_outputs = control_flow_ops.while_loop(
cond=lambda i, _1, _2, _3: i < max_frames,
body=_pretrain_backward,
loop_vars=(tf.constant(0, dtype=tf.int32), h1, h0,backward_outputs_state))
backward_state_outputs = backward_state_outputs.stack()
batch_size = tf.shape(model_input)[0]
index_1 = tf.range(0, batch_size) * max_frames + (num_frames - 1)
backward_state_outputs = tf.transpose(backward_state_outputs, [1, 0, 2])
backward_state_outputs = tf.gather(tf.reshape(backward_state_outputs, [-1, hidden_dim]), index_1)
moe_input = tf.concat((forward_state_outputs, backward_state_outputs), axis=1)
final_probabilities = self.sub_moe(moe_input,vocab_size,scopename="backward")
#probabilities_by_class = self.sub_moe(forward_state_outputs,vocab_size,scopename="forward")
return {"predictions": final_probabilities}
def create_forward_recurrent_unit(self,emb_dim,hidden_dim,l2_penalty):
# Weights and Bias for input and hidden tensor
Wi = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wi))
Ui = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Ui")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Ui))
Vi = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Vi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vi))
bi = tf.Variable(tf.constant(0.1, shape=[1]), name="bi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bi))
Wf = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wf))
Vf = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Vf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vf))
Uf = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Uf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uf))
bf = tf.Variable(tf.constant(0.1, shape=[1]), name="bf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bf))
Wog = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wog))
Vog = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Vog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vog))
Uog = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uog))
bog = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bog))
Wc = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wc))
Vc = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Vc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vc))
Uc = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uc))
bc = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bc))
Wix = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wix")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wix))
Vix = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Vix")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vix))
Uix = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Uix")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uix))
bix = tf.Variable(tf.constant(0.1, shape=[1]), name="bix")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bix))
Wfx = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wfx")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wfx))
Vfx = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Vfx")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vfx))
Ufx = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Ufx")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Ufx))
bfx = tf.Variable(tf.constant(0.1, shape=[1]), name="bfx")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bfx))
def unit(x, hidden_memory_tm0, hidden_memory_tm1):
previous_hidden_x, x_prev = tf.unstack(hidden_memory_tm0)
previous_hidden_state, c_prev = tf.unstack(hidden_memory_tm1)
# Input Gate
i = tf.sigmoid(
tf.matmul(x, Wi) + tf.matmul(previous_hidden_x, Vi) +
tf.matmul(previous_hidden_state, Ui) + bi
)
# Forget Gate
f = tf.sigmoid(
tf.matmul(x, Wf) + tf.matmul(previous_hidden_x, Vf) +
tf.matmul(previous_hidden_state, Uf) + bf
)
# Input Gate
ix = tf.sigmoid(
tf.matmul(x, Wix) + tf.matmul(previous_hidden_x, Vix) +
tf.matmul(previous_hidden_state, Uix) + bix
)
# Forget Gate
fx = tf.sigmoid(
tf.matmul(x, Wfx) + tf.matmul(previous_hidden_x, Vfx) +
tf.matmul(previous_hidden_state, Ufx) + bfx
)
# Output Gate
o = tf.sigmoid(
tf.matmul(x, Wog) + tf.matmul(previous_hidden_x, Vog) +
tf.matmul(previous_hidden_state, Uog) + bog
)
# New Memory Cell
c_ = tf.tanh(
tf.matmul(x, Wc) + tf.matmul(previous_hidden_x, Vc) +
tf.matmul(previous_hidden_state, Uc) + bc
)
# Final Memory cell
c = f * c_prev + i * c_
current_x = fx * x_prev + ix * x
current_hidden_x = tf.nn.l2_normalize(current_x, dim=1)
# Current Hidden state
current_hidden_state = o * tf.tanh(c)
return f, tf.stack([current_hidden_x, current_x]), tf.stack([current_hidden_state, c])
return unit
def create_backward_recurrent_unit(self,emb_dim,hidden_dim,l2_penalty):
# Weights and Bias for input and hidden tensor
Wi = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wi))
Ui = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Ui")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Ui))
Vi = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Vi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vi))
Xi = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Xi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Xi))
bi = tf.Variable(tf.constant(0.1, shape=[1]), name="bi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bi))
Wf = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wf))
Vf = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Vf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vf))
Uf = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Uf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uf))
Xf = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Xf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Xf))
bf = tf.Variable(tf.constant(0.1, shape=[1]), name="bf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bf))
Wog = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wog))
Vog = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Vog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vog))
Uog = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uog))
Xog = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Xog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Xog))
bog = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bog))
Wc = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wc))
Vc = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Vc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vc))
Uc = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uc))
Xc = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Xc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Xc))
bc = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bc))
Wix = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wix")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wix))
Vix = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Vix")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vix))
Uix = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Uix")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uix))
Xix = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Xix")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Xix))
bix = tf.Variable(tf.constant(0.1, shape=[1]), name="bix")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bix))
Wfx = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wfx")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wfx))
Vfx = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Vfx")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Vfx))
Ufx = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Ufx")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Ufx))
Xfx = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Xfx")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Xfx))
bfx = tf.Variable(tf.constant(0.1, shape=[1]), name="bfx")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bfx))
def unit(x, y, hidden_memory_tm0, hidden_memory_tm1):
previous_hidden_x, x_prev = tf.unstack(hidden_memory_tm0)
previous_hidden_state, c_prev = tf.unstack(hidden_memory_tm1)
# Input Gate
i = tf.sigmoid(
tf.matmul(x, Wi) + tf.matmul(previous_hidden_x, Vi) + tf.matmul(y, Xi) +
tf.matmul(previous_hidden_state, Ui) + bi
)
# Forget Gate
f = tf.sigmoid(
tf.matmul(x, Wf) + tf.matmul(previous_hidden_x, Vf) + tf.matmul(y, Xf) +
tf.matmul(previous_hidden_state, Uf) + bf
)
# Input Gate
ix = tf.sigmoid(
tf.matmul(x, Wix) + tf.matmul(previous_hidden_x, Vix) + tf.matmul(y, Xix) +
tf.matmul(previous_hidden_state, Uix) + bix
)
# Forget Gate
fx = tf.sigmoid(
tf.matmul(x, Wfx) + tf.matmul(previous_hidden_x, Vfx) + tf.matmul(y, Xfx) +
tf.matmul(previous_hidden_state, Ufx) + bfx
)
# Output Gate
o = tf.sigmoid(
tf.matmul(x, Wog) + tf.matmul(previous_hidden_x, Vog) + tf.matmul(y, Xog) +
tf.matmul(previous_hidden_state, Uog) + bog
)
# New Memory Cell
c_ = tf.tanh(
tf.matmul(x, Wc) + tf.matmul(previous_hidden_x, Vc) + tf.matmul(y, Xc) +
tf.matmul(previous_hidden_state, Uc) + bc
)
# Final Memory cell
c = f * c_prev + i * c_
current_x = fx * x_prev + ix * x
current_hidden_x = tf.nn.l2_normalize(current_x, dim=1)
# Current Hidden state
current_hidden_state = o * tf.tanh(c)
return f, tf.stack([current_hidden_x, current_x]), tf.stack([current_hidden_state, c])
return unit
def sub_moe(self,model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
scopename="",
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+scopename)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts"+scopename)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
class LstmGateModel(models.BaseModel):
"""Logistic model with L2 regularization."""
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
"""Creates a matrix regression model.
Args:
model_input: 'batch' x 'num_features' x 'num_methods' matrix of input features.
vocab_size: The number of classes in the dataset.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes."""
lstm_size = FLAGS.lstm_cells
max_frames = model_input.get_shape().as_list()[1]
emb_dim = model_input.get_shape().as_list()[2]
hidden_dim = lstm_size
with tf.variable_scope('lstm_forward'):
g_recurrent_unit_forward = self.create_recurrent_unit(emb_dim,hidden_dim,l2_penalty)
h0 = tf.zeros([tf.shape(model_input)[0], hidden_dim])
h0 = tf.stack([h0, h0])
outputs_gate = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
outputs_state = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
def _pretrain_forward(i,h_tm1,g_predictions,s_predictions):
x_t = model_input[:,i,:]
gate, h_t = g_recurrent_unit_forward(x_t, h_tm1)
hidden_state, c_prev = tf.unstack(h_t)
g_predictions = g_predictions.write(i,gate)
s_predictions = s_predictions.write(i,c_prev)
return i + 1, h_t, g_predictions, s_predictions
_, _, gate_outputs, state_outputs = control_flow_ops.while_loop(
cond=lambda i, _1, _2, _3: i < max_frames,
body=_pretrain_forward,
loop_vars=(tf.constant(0, dtype=tf.int32),h0,outputs_gate,outputs_state))
gate_outputs = gate_outputs.stack()
state_outputs = state_outputs.stack()
batch_size = tf.shape(model_input)[0]
index_1 = tf.range(0, batch_size) * max_frames + (num_frames - 1)
gate_outputs = tf.transpose(gate_outputs, [1, 0, 2])
gate_outputs = tf.gather(tf.reshape(gate_outputs, [-1, hidden_dim]), index_1)
state_outputs = tf.transpose(state_outputs, [1, 0, 2])
state_outputs = tf.gather(tf.reshape(state_outputs, [-1, hidden_dim]), index_1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=state_outputs,
vocab_size=vocab_size,
**unused_params)
def create_recurrent_unit(self,emb_dim,hidden_dim,l2_penalty):
# Weights and Bias for input and hidden tensor
Wi = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wi))
Ui = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Ui")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Ui))
bi = tf.Variable(tf.constant(0.1, shape=[1]), name="bi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bi))
Wf = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wf))
Uf = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Uf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uf))
bf = tf.Variable(tf.constant(1.0, shape=[1]), name="bf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bf))
Wog = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wog))
Uog = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uog))
bog = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bog))
Wc = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wc))
Uc = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uc))
bc = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bc))
def unit(x, hidden_memory_tm1):
previous_hidden_state, c_prev = tf.unstack(hidden_memory_tm1)
# Input Gate
i = tf.sigmoid(
tf.matmul(x, Wi) +
tf.matmul(previous_hidden_state, Ui) + bi
)
# Forget Gate
f = tf.sigmoid(
tf.matmul(x, Wf) +
tf.matmul(previous_hidden_state, Uf) + bf
)
# Output Gate
o = tf.sigmoid(
tf.matmul(x, Wog) +
tf.matmul(previous_hidden_state, Uog) + bog
)
# New Memory Cell
c_ = tf.nn.tanh(
tf.matmul(x, Wc)+
tf.matmul(previous_hidden_state, Uc) + bc
)
# Final Memory cell
c = f * c_prev + i * c_
# Current Hidden state
current_hidden_state = o * tf.nn.tanh(c)
return f, tf.stack([current_hidden_state, c])
return unit
class LstmGateMultilayerModel(models.BaseModel):
"""Logistic model with L2 regularization."""
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
"""Creates a matrix regression model.
Args:
model_input: 'batch' x 'num_features' x 'num_methods' matrix of input features.
vocab_size: The number of classes in the dataset.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes."""
lstm_size = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
hidden_outputs = model_input
state_outputs = []
for i in range(number_of_layers):
state_output, hidden_outputs = self.rnn_gate(hidden_outputs, lstm_size, num_frames, sub_scope="lstm_lsyer%d" % i)
state_outputs.append(state_output)
state_outputs = tf.concat(state_outputs,axis=1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=state_outputs,
vocab_size=vocab_size,
**unused_params)
def rnn_gate(self, model_input, lstm_size, num_frames, l2_penalty=1e-8, sub_scope="", **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
## Batch normalize the input
max_frames = model_input.get_shape().as_list()[1]
emb_dim = model_input.get_shape().as_list()[2]
hidden_dim = lstm_size
with tf.variable_scope(sub_scope+'lstm_forward'):
g_recurrent_unit_forward = self.create_recurrent_unit(emb_dim,hidden_dim,l2_penalty)
h0 = tf.zeros([tf.shape(model_input)[0], hidden_dim])
h0 = tf.stack([h0, h0])
outputs_hidden = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
outputs_state = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
def _pretrain_forward(i,h_tm1,s_predictions,h_predictions):
x_t = model_input[:,i,:]
gate, h_t = g_recurrent_unit_forward(x_t, h_tm1)
hidden_state, c_prev = tf.unstack(h_t)
h_predictions = h_predictions.write(i,hidden_state)
s_predictions = s_predictions.write(i,c_prev)
return i + 1, h_t, s_predictions, h_predictions
_, _, state_outputs, hidden_outputs = control_flow_ops.while_loop(
cond=lambda i, _1, _2, _3: i < max_frames,
body=_pretrain_forward,
loop_vars=(tf.constant(0, dtype=tf.int32), h0, outputs_state, outputs_hidden),
swap_memory=True)
state_outputs = state_outputs.stack()
state_outputs = tf.transpose(state_outputs, [1, 0, 2])
hidden_outputs = hidden_outputs.stack()
hidden_outputs = tf.transpose(hidden_outputs, [1, 0, 2])
hidden_outputs = tf.reshape(hidden_outputs, [-1, max_frames, lstm_size])
batch_size = tf.shape(model_input)[0]
index_1 = tf.range(0, batch_size) * max_frames + (num_frames - 1)
state_outputs = tf.gather(tf.reshape(state_outputs, [-1, hidden_dim]), index_1)
return state_outputs, hidden_outputs
def create_recurrent_unit(self,emb_dim,hidden_dim,l2_penalty):
# Weights and Bias for input and hidden tensor
Wi = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wi))
Ui = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Ui")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Ui))
bi = tf.Variable(tf.constant(0.1, shape=[1]), name="bi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bi))
Wf = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wf))
Uf = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Uf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uf))
bf = tf.Variable(tf.constant(1.0, shape=[1]), name="bf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bf))
Wog = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wog))
Uog = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uog))
bog = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bog))
Wc = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wc))
Uc = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uc))
bc = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bc))
def unit(x, hidden_memory_tm1):
previous_hidden_state, c_prev = tf.unstack(hidden_memory_tm1)
# Input Gate
i = tf.sigmoid(
tf.matmul(x, Wi) +
tf.matmul(previous_hidden_state, Ui) + bi
)
# Forget Gate
f = tf.sigmoid(
tf.matmul(x, Wf) +
tf.matmul(previous_hidden_state, Uf) + bf
)
# Output Gate
o = tf.sigmoid(
tf.matmul(x, Wog) +
tf.matmul(previous_hidden_state, Uog) + bog
)
# New Memory Cell
c_ = tf.nn.tanh(
tf.matmul(x, Wc)+
tf.matmul(previous_hidden_state, Uc) + bc
)
# Final Memory cell
c = f * c_prev + i * c_
# Current Hidden state
current_hidden_state = o * tf.nn.tanh(c)
return f, tf.stack([current_hidden_state, c])
return unit
class LstmQuickMemoryModel(models.BaseModel):
"""Logistic model with L2 regularization."""
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
"""Creates a matrix regression model.
Args:
model_input: 'batch' x 'num_features' x 'num_methods' matrix of input features.
vocab_size: The number of classes in the dataset.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes."""
lstm_size = FLAGS.lstm_cells
max_frames = model_input.get_shape().as_list()[1]
emb_dim = model_input.get_shape().as_list()[2]
hidden_dim = lstm_size
with tf.variable_scope('lstm_forward'):
g_recurrent_unit_forward = self.create_recurrent_unit(emb_dim,hidden_dim,l2_penalty)
h0 = tf.zeros([tf.shape(model_input)[0], hidden_dim])
h0 = tf.stack([h0, h0, h0])
outputs_state = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
def _pretrain_forward(i, h_tm1, s_predictions):
x_t = model_input[:,i,:]
gate, h_t1 = g_recurrent_unit_forward(x_t, h_tm1)
hidden_state, c_prev, m_prev = tf.unstack(h_t1)
s_predictions = s_predictions.write(i, c_prev + m_prev)
return i + 1, h_t1, s_predictions
_, _, state_outputs = control_flow_ops.while_loop(
cond=lambda i, _1, _2: i < max_frames,
body=_pretrain_forward,
loop_vars=(tf.constant(0, dtype=tf.int32), h0,outputs_state))
state_outputs = state_outputs.stack()
batch_size = tf.shape(model_input)[0]
index_1 = tf.range(0, batch_size) * max_frames + (num_frames - 1)
state_outputs = tf.transpose(state_outputs, [1, 0, 2])
state_outputs = tf.gather(tf.reshape(state_outputs, [-1, hidden_dim]), index_1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=state_outputs,
vocab_size=vocab_size,
**unused_params)
def create_recurrent_unit(self,emb_dim,hidden_dim,l2_penalty):
# Weights and Bias for input and hidden tensor
Wi = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wi))
Ui = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Ui")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Ui))
bi = tf.Variable(tf.constant(0.1, shape=[1]), name="bi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bi))
Wf = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wf))
Uf = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Uf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uf))
bf = tf.Variable(tf.constant(0.1, shape=[1]), name="bf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bf))
Wog = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wog))
Uog = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uog))
bog = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bog))
Wc = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wc))
Uc = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uc))
bc = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bc))
Wom = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wom")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wom))
Uom = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uom")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uom))
bom = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bom")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bom))
Wfm = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wfm")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wfm))
Ufm = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Ufm")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Ufm))
bfm = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bfm")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bfm))
Wm = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wm")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wm))
Um = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Um")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Um))
bm = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bm")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bm))
def unit(x, hidden_memory_tm1):
previous_hidden_state, c_prev, m_prev = tf.unstack(hidden_memory_tm1)
# Input Gate
i = tf.sigmoid(
tf.matmul(x, Wi) +
tf.matmul(previous_hidden_state, Ui) + bi
)
# Forget Gate
f = tf.sigmoid(
tf.matmul(x, Wf) +
tf.matmul(previous_hidden_state, Uf) + bf
)
# Output Gate
o = tf.sigmoid(
tf.matmul(x, Wog) +
tf.matmul(previous_hidden_state, Uog) + bog
)
# long Gate
im = tf.sigmoid(
tf.matmul(x, Wom) +
tf.matmul(previous_hidden_state, Uom) + bom
)
# long Gate
fm = tf.sigmoid(
tf.matmul(x, Wfm) +
tf.matmul(previous_hidden_state, Ufm) + bfm
)
# New Memory Cell
c_ = tf.tanh(
tf.matmul(x, Wc) +
tf.matmul(previous_hidden_state, Uc) + bc
)
# New Memory Cell
m_ = tf.tanh(
tf.matmul(x, Wm) +
tf.matmul(previous_hidden_state, Um) + bm
)
# Final Memory cell
m = im * m_ + fm * m_prev
c = f * c_prev + i * c_
# Current Hidden state
current_hidden_state = o * tf.tanh(c + m)
return f, tf.stack([current_hidden_state, c, m])
return unit
class LstmLinearOutputModel(models.BaseModel):
"""Logistic model with L2 regularization."""
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
"""Creates a matrix regression model.
Args:
model_input: 'batch' x 'num_features' x 'num_methods' matrix of input features.
vocab_size: The number of classes in the dataset.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes."""
lstm_size = FLAGS.lstm_cells
max_frames = model_input.get_shape().as_list()[1]
emb_dim = model_input.get_shape().as_list()[2]
hidden_dim = lstm_size
with tf.variable_scope('lstm_forward'):
g_recurrent_unit_forward = self.create_recurrent_unit(emb_dim,hidden_dim,l2_penalty)
h0 = tf.zeros([tf.shape(model_input)[0], hidden_dim])
h0 = tf.stack([h0, h0])
outputs_gate = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
outputs_state = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
def _pretrain_forward(i,h_tm1,g_predictions,s_predictions):
x_t = model_input[:,i,:]
gate, h_t = g_recurrent_unit_forward(x_t, h_tm1)
hidden_state, c_prev = tf.unstack(h_t)
g_predictions = g_predictions.write(i,gate)
s_predictions = s_predictions.write(i,c_prev)
return i + 1, h_t, g_predictions, s_predictions
_, _, gate_outputs, state_outputs = control_flow_ops.while_loop(
cond=lambda i, _1, _2, _3: i < max_frames,
body=_pretrain_forward,
loop_vars=(tf.constant(0, dtype=tf.int32),h0,outputs_gate,outputs_state))
gate_outputs = gate_outputs.stack()
state_outputs = state_outputs.stack()
batch_size = tf.shape(model_input)[0]
index_1 = tf.range(0, batch_size) * max_frames + (num_frames - 1)
gate_outputs = tf.transpose(gate_outputs, [1, 0, 2])
gate_outputs = tf.gather(tf.reshape(gate_outputs, [-1, hidden_dim]), index_1)
state_outputs = tf.transpose(state_outputs, [1, 0, 2])
state_outputs = tf.gather(tf.reshape(state_outputs, [-1, hidden_dim]), index_1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=state_outputs,
vocab_size=vocab_size,
**unused_params)
def create_recurrent_unit(self,emb_dim,hidden_dim,l2_penalty):
# Weights and Bias for input and hidden tensor
Wi = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wi))
Ui = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Ui")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Ui))
bi = tf.Variable(tf.constant(0.1, shape=[1]), name="bi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bi))
Wf = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wf))
Uf = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Uf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uf))
bf = tf.Variable(tf.constant(0.1, shape=[1]), name="bf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bf))
Wog = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wog))
Uog = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uog))
bog = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bog))
Wc = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wc))
Uc = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uc))
bc = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bc))
def unit(x, hidden_memory_tm1):
previous_hidden_state, c_prev = tf.unstack(hidden_memory_tm1)
# Input Gate
i = tf.sigmoid(
tf.matmul(x, Wi) +
tf.matmul(previous_hidden_state, Ui) + bi
)
# Forget Gate
f = tf.sigmoid(
tf.matmul(x, Wf) +
tf.matmul(previous_hidden_state, Uf) + bf
)
# Output Gate
o = tf.sigmoid(
tf.matmul(x, Wog) +
tf.matmul(previous_hidden_state, Uog) + bog
)
# New Memory Cell
c_ = tf.nn.tanh(
tf.matmul(x, Wc)+
tf.matmul(previous_hidden_state, Uc) + bc
)
# Final Memory cell
c = f * c_prev + i * c_
# Current Hidden state
current_hidden_state = o * tf.nn.l2_normalize(c,dim=1)
return f, tf.stack([current_hidden_state, c])
return unit
class LstmLinearOutput2Model(models.BaseModel):
"""Logistic model with L2 regularization."""
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
"""Creates a matrix regression model.
Args:
model_input: 'batch' x 'num_features' x 'num_methods' matrix of input features.
vocab_size: The number of classes in the dataset.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes."""
lstm_size = FLAGS.lstm_cells
max_frames = model_input.get_shape().as_list()[1]
emb_dim = model_input.get_shape().as_list()[2]
hidden_dim = lstm_size
with tf.variable_scope('lstm_forward'):
g_recurrent_unit_forward = self.create_recurrent_unit(emb_dim,hidden_dim,l2_penalty)
h0 = tf.zeros([tf.shape(model_input)[0], hidden_dim])
h0 = tf.stack([h0, h0])
outputs_gate = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
outputs_state = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
def _pretrain_forward(i,h_tm1,g_predictions,s_predictions):
x_t = model_input[:,i,:]
gate, h_t = g_recurrent_unit_forward(x_t, h_tm1)
hidden_state, c_prev = tf.unstack(h_t)
g_predictions = g_predictions.write(i,gate)
s_predictions = s_predictions.write(i,c_prev)
return i + 1, h_t, g_predictions, s_predictions
_, _, gate_outputs, state_outputs = control_flow_ops.while_loop(
cond=lambda i, _1, _2, _3: i < max_frames,
body=_pretrain_forward,
loop_vars=(tf.constant(0, dtype=tf.int32),h0,outputs_gate,outputs_state))
gate_outputs = gate_outputs.stack()
state_outputs = state_outputs.stack()
batch_size = tf.shape(model_input)[0]
index_1 = tf.range(0, batch_size) * max_frames + (num_frames - 1)
gate_outputs = tf.transpose(gate_outputs, [1, 0, 2])
gate_outputs = tf.gather(tf.reshape(gate_outputs, [-1, hidden_dim]), index_1)
state_outputs = tf.transpose(state_outputs, [1, 0, 2])
state_outputs = tf.gather(tf.reshape(state_outputs, [-1, hidden_dim]), index_1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=state_outputs,
vocab_size=vocab_size,
**unused_params)
def create_recurrent_unit(self,emb_dim,hidden_dim,l2_penalty):
# Weights and Bias for input and hidden tensor
Wi = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wi))
Ui = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Ui")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Ui))
bi = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bi))
Wf = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wf))
Uf = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uf))
bf = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bf))
Wog = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wog))
Uog = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uog))
bog = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bog")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bog))
Wc = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wc))
Uc = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uc))
bc = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bc))
def unit(x, hidden_memory_tm1):
previous_hidden_state, c_prev = tf.unstack(hidden_memory_tm1)
# Input Gate
i = tf.sigmoid(
tf.matmul(x, Wi) +
tf.matmul(previous_hidden_state, Ui) + bi
)
# Forget Gate
f = tf.sigmoid(
tf.matmul(x, Wf) +
tf.matmul(previous_hidden_state, Uf) + bf + 1.0
)
# Output Gate
o = tf.sigmoid(
tf.matmul(x, Wog) +
tf.matmul(previous_hidden_state, Uog) + bog
)
# New Memory Cell
c_ = tf.nn.tanh(
tf.matmul(x, Wc) +
tf.matmul(previous_hidden_state, Uc) + bc
)
# Final Memory cell
c = f * c_prev + i * c_
# Current Hidden state
current_hidden_state = o * tf.nn.l2_normalize(c,dim=1)
return f, tf.stack([current_hidden_state, c])
return unit
class LstmNoOutputModel(models.BaseModel):
"""Logistic model with L2 regularization."""
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
"""Creates a matrix regression model.
Args:
model_input: 'batch' x 'num_features' x 'num_methods' matrix of input features.
vocab_size: The number of classes in the dataset.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes."""
lstm_size = FLAGS.lstm_cells
max_frames = model_input.get_shape().as_list()[1]
emb_dim = model_input.get_shape().as_list()[2]
hidden_dim = lstm_size
with tf.variable_scope('lstm_forward'):
g_recurrent_unit_forward = self.create_recurrent_unit(emb_dim,hidden_dim,l2_penalty)
h0 = tf.zeros([tf.shape(model_input)[0], hidden_dim])
h0 = tf.stack([h0, h0])
outputs_gate = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
outputs_state = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
def _pretrain_forward(i,h_tm1,g_predictions,s_predictions):
x_t = model_input[:,i,:]
gate, h_t = g_recurrent_unit_forward(x_t, h_tm1)
hidden_state, c_prev = tf.unstack(h_t)
g_predictions = g_predictions.write(i,gate)
s_predictions = s_predictions.write(i,c_prev)
return i + 1, h_t, g_predictions, s_predictions
_, _, gate_outputs, state_outputs = control_flow_ops.while_loop(
cond=lambda i, _1, _2, _3: i < max_frames,
body=_pretrain_forward,
loop_vars=(tf.constant(0, dtype=tf.int32),h0,outputs_gate,outputs_state))
gate_outputs = gate_outputs.stack()
state_outputs = state_outputs.stack()
batch_size = tf.shape(model_input)[0]
index_1 = tf.range(0, batch_size) * max_frames + (num_frames - 1)
gate_outputs = tf.transpose(gate_outputs, [1, 0, 2])
gate_outputs = tf.gather(tf.reshape(gate_outputs, [-1, hidden_dim]), index_1)
state_outputs = tf.transpose(state_outputs, [1, 0, 2])
state_outputs = tf.gather(tf.reshape(state_outputs, [-1, hidden_dim]), index_1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=state_outputs,
vocab_size=vocab_size,
**unused_params)
def create_recurrent_unit(self,emb_dim,hidden_dim,l2_penalty):
# Weights and Bias for input and hidden tensor
Wi = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wi))
Ui = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Ui")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Ui))
bi = tf.Variable(tf.constant(0.1, shape=[1]), name="bi")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bi))
Wf = tf.Variable(tf.truncated_normal([emb_dim, 1], stddev=0.1), name="Wf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wf))
Uf = tf.Variable(tf.truncated_normal([hidden_dim, 1], stddev=0.1), name="Uf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uf))
bf = tf.Variable(tf.constant(0.1, shape=[1]), name="bf")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bf))
Wc = tf.Variable(tf.truncated_normal([emb_dim, hidden_dim], stddev=0.1), name="Wc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Wc))
Uc = tf.Variable(tf.truncated_normal([hidden_dim, hidden_dim], stddev=0.1), name="Uc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Uc))
bc = tf.Variable(tf.constant(0.1, shape=[hidden_dim]), name="bc")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(bc))
def unit(x, hidden_memory_tm1):
previous_hidden_state, c_prev = tf.unstack(hidden_memory_tm1)
# Input Gate
i = tf.sigmoid(
tf.matmul(x, Wi) +
tf.matmul(previous_hidden_state, Ui) + bi
)
# Forget Gate
f = tf.sigmoid(
tf.matmul(x, Wf) +
tf.matmul(previous_hidden_state, Uf) + bf
)
# New Memory Cell
c_ = tf.nn.tanh(
tf.matmul(x, Wc)+
tf.matmul(previous_hidden_state, Uc) + bc
)
# Final Memory cell
c = f * c_prev + i * c_
# Current Hidden state
current_hidden_state = tf.nn.l2_normalize(c,dim=1)
return f, tf.stack([current_hidden_state, c])
return unit
class LstmModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32)
state_c = tf.concat(map(lambda x: x.c, state), axis=1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=state_c,
vocab_size=vocab_size,
**unused_params)
class LstmDiffModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
shape = model_input.get_shape().as_list()
frames_sum = tf.reduce_sum(tf.abs(model_input),axis=2)
frames_true = tf.ones(tf.shape(frames_sum))
frames_false = tf.zeros(tf.shape(frames_sum))
frames_bool = tf.reshape(tf.where(tf.greater(frames_sum, frames_false), frames_true, frames_false),[-1,shape[1],1])
model_mean = tf.reduce_sum(model_input*frames_bool,axis=1,keep_dims=True)/tf.reduce_sum(frames_bool,axis=1,keep_dims=True)
model_input = (model_input - model_mean)*frames_bool
model_input = tf.nn.l2_normalize(model_input, 2)
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32)
state_c = tf.concat(map(lambda x: x.c, state), axis=1)
model_mean = tf.nn.l2_normalize(tf.reshape(model_mean,[-1,shape[2]]), dim=1)
rnn_out = tf.nn.l2_normalize(state_c, dim=1)
model_output = tf.concat((model_mean,rnn_out),axis=1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=model_output,
vocab_size=vocab_size,
**unused_params)
class LstmFramesModel(models.BaseModel):
def sub_moe(self,model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
scopename="",
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
num_extends = FLAGS.moe_layers
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+scopename)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts"+scopename)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reduce_mean(tf.reshape(final_probabilities_by_class_and_batch,
[-1, num_extends, vocab_size]),axis=1)
return {"predictions": final_probabilities}
def calculate_loss(self, predictions, labels, **unused_params):
with tf.name_scope("loss_frames"):
epsilon = 10e-6
float_labels = tf.cast(labels, tf.float32)
cross_entropy_loss = float_labels * tf.log(predictions + epsilon) + (
1 - float_labels) * tf.log(1 - predictions + epsilon)
return tf.reduce_sum(cross_entropy_loss, 2)
def create_model(self, model_input, vocab_size, num_frames,l2_penalty=1e-8, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
shape = model_input.get_shape().as_list()
frames_sum = tf.reduce_sum(tf.abs(model_input),axis=2)
frames_true = tf.ones(tf.shape(frames_sum))
frames_false = tf.zeros(tf.shape(frames_sum))
frames_bool = tf.reshape(tf.where(tf.greater(frames_sum, frames_false), frames_true, frames_false),[-1,shape[1],1])
unit_layer_1 = FLAGS.stride_size
unit_layer_2 = shape[1]//FLAGS.stride_size
model_input = model_input[:,:unit_layer_1*unit_layer_2,:]
model_input_1 = tf.reshape(model_input,[-1,unit_layer_1,shape[2]])
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
with tf.variable_scope("RNN-1"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input_1,
sequence_length=None,
swap_memory=True,
dtype=tf.float32)
## Batch normalize the input
sigmoid_input = tf.concat(map(lambda x: x.c, state), axis=1)
frames_bool = frames_bool[:,0:shape[1]:FLAGS.stride_size,:]
probabilities_by_batch = slim.fully_connected(
sigmoid_input,
vocab_size,
activation_fn=tf.nn.sigmoid,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="expert_activations")
probabilities_by_frame = tf.reshape(probabilities_by_batch,[-1,shape[1]//FLAGS.stride_size,vocab_size])*frames_bool
probabilities_transpose = tf.transpose(probabilities_by_frame,[0,2,1])
probabilities_topk, _ = tf.nn.top_k(probabilities_transpose, k=FLAGS.moe_layers)
probabilities_topk = tf.reshape(tf.transpose(probabilities_topk, [0,2,1]),[-1,vocab_size*FLAGS.moe_layers])
importance_by_frame = self.calculate_loss(probabilities_by_frame,tf.reduce_mean(
tf.reshape(probabilities_topk,[-1,FLAGS.moe_layers,vocab_size]),axis=1,keep_dims=True))
_, index_topk = tf.nn.top_k(importance_by_frame, k=FLAGS.moe_layers)
batch_size = tf.shape(model_input)[0]
batch_index = tf.tile(tf.expand_dims(tf.range(batch_size), 1), [1, FLAGS.moe_layers])
index = tf.reshape(batch_index*(shape[1]//FLAGS.stride_size) + index_topk,[-1])
moe_input = tf.reshape(tf.gather(sigmoid_input, index),[-1,FLAGS.moe_layers,lstm_size*number_of_layers])
result = self.sub_moe(moe_input,vocab_size)
result["predictions_class"] = probabilities_topk
return result
class LstmFrames2Model(models.BaseModel):
def sub_moe(self,model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
scopename="",
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+scopename)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts"+scopename)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
def calculate_loss(self, predictions, labels, **unused_params):
with tf.name_scope("loss_frames"):
epsilon = 10e-6
float_labels = tf.cast(labels, tf.float32)
cross_entropy_loss = float_labels * tf.log(predictions + epsilon) + (
1 - float_labels) * tf.log(1 - predictions + epsilon)
return tf.reduce_sum(cross_entropy_loss, axis=2)
def create_model(self, model_input, vocab_size, num_frames,l2_penalty=1e-8, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
shape = model_input.get_shape().as_list()
frames_sum = tf.reduce_sum(tf.abs(model_input),axis=2)
frames_true = tf.ones(tf.shape(frames_sum))
frames_false = tf.zeros(tf.shape(frames_sum))
frames_bool = tf.reshape(tf.where(tf.greater(frames_sum, frames_false), frames_true, frames_false),[-1,shape[1],1])
model_mean = tf.reduce_sum(model_input*frames_bool, axis=1)/tf.reduce_sum(frames_bool, axis=1)
video_probabilities = self.sub_moe(model_mean, vocab_size, scopename="_video")
unit_layer_1 = FLAGS.stride_size
unit_layer_2 = shape[1]//FLAGS.stride_size
model_input = model_input[:,:unit_layer_1*unit_layer_2,:]
model_input_1 = tf.reshape(model_input,[-1,unit_layer_1,shape[2]])
sigmoid_input = tf.reduce_mean(model_input_1,axis=1)
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
with tf.variable_scope("RNN-1"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input_1,
sequence_length=None,
swap_memory=True,
dtype=tf.float32)
## Batch normalize the input
state_out = tf.concat(map(lambda x: x.c, state), axis=1)
probabilities_by_batch = slim.fully_connected(
sigmoid_input,
vocab_size,
activation_fn=tf.nn.sigmoid,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="expert_activations")
probabilities_by_frame = tf.reshape(probabilities_by_batch,[-1,unit_layer_2,vocab_size])
importance_by_frame = self.calculate_loss(probabilities_by_frame, tf.reshape(video_probabilities,[-1,1,vocab_size]))
_, index_topk = tf.nn.top_k(importance_by_frame, k=FLAGS.moe_num_extend)
batch_size = tf.shape(model_input)[0]
batch_index = tf.tile(tf.expand_dims(tf.range(batch_size), 1), [1, FLAGS.moe_num_extend])
index = tf.reshape(batch_index*unit_layer_2 + index_topk,[-1])
moe_input = tf.gather(state_out, index)
frame_probabilities = self.sub_moe(moe_input,vocab_size,scopename="_frame")
result = {}
result["prediction_frames"] = frame_probabilities
result["prediction_prepare_frames"] = tf.reshape(probabilities_by_frame,[-1,vocab_size])
result["prediction_prepare_video"] = video_probabilities
result["predictions"] = tf.reduce_mean(tf.reshape(frame_probabilities,[-1,FLAGS.moe_num_extend,vocab_size]),axis=1)
return result
class LstmFrames3Model(models.BaseModel):
def sub_moe(self,model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
scopename="",
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+scopename)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts"+scopename)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
def create_model(self, model_input, vocab_size, num_frames,l2_penalty=1e-8, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
shape = model_input.get_shape().as_list()
num_extend = FLAGS.moe_num_extend - 1
unit_layer_1 = FLAGS.stride_size
unit_layer_2 = shape[1]//FLAGS.stride_size
model_input = model_input[:,:unit_layer_1*unit_layer_2,:]
num_frames = tf.maximum(num_frames//FLAGS.stride_size//num_extend, 1)
model_input_1 = tf.reshape(model_input,[-1,unit_layer_1,shape[2]])
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
with tf.variable_scope("RNN-1"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input_1,
sequence_length=None,
swap_memory=True,
dtype=tf.float32)
## Batch normalize the input
state_out = tf.concat(map(lambda x: x.c, state), axis=1)
random_vector = tf.random_uniform([lstm_size*number_of_layers,1],-1,1)
random_out = tf.reshape(tf.matmul(state_out,random_vector),[-1,unit_layer_2])
_, indexes = tf.nn.top_k(random_out, sorted=True, k=unit_layer_2)
flat_state_out = tf.reshape(state_out,[-1,unit_layer_2,lstm_size*number_of_layers])
divide_inputs = []
length = unit_layer_2//num_extend
frame_bool = tf.reshape(tf.sequence_mask(num_frames,maxlen=length,dtype=tf.float32),[-1,length,1])
for i in range(num_extend):
begin_frames = tf.reshape(num_frames*i,[-1,1])
frames_index = tf.reshape(tf.range(length),[1,length])
frames_index = begin_frames + frames_index
batch_size = tf.shape(model_input)[0]
batch_index = tf.tile(
tf.expand_dims(tf.range(batch_size), 1), [1, length])
index = tf.stack([batch_index, tf.cast(frames_index,dtype=tf.int32)], 2)
divide_index = tf.gather_nd(indexes, index)
batch_features_index = tf.stack([batch_index, tf.cast(divide_index,dtype=tf.int32)], 2)
divide_feature = tf.gather_nd(flat_state_out, batch_features_index)
divide_feature = tf.reduce_sum(divide_feature*frame_bool,axis=1)/(tf.reduce_sum(frame_bool,axis=1)+1e-6)
divide_inputs.append(divide_feature)
moe_input = tf.reshape(tf.stack(divide_inputs,axis=1),[-1,lstm_size*number_of_layers])
frame_probabilities = self.sub_moe(moe_input,vocab_size,scopename="_frame")
result = {}
flat_frame_probabilities = tf.reshape(frame_probabilities,[-1,num_extend,vocab_size])
softmax_activations = slim.fully_connected(
moe_input,
vocab_size,
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="softmax")
softmax_distribution = tf.nn.softmax(tf.reshape(
softmax_activations,
[-1,num_extend,vocab_size]),dim=1) # (Batch * #Labels) x (num_mixtures + 1)
final_probabilities = tf.reduce_sum(
softmax_distribution * flat_frame_probabilities, axis=1)
result["predictions"] = final_probabilities
result["prediction_frames"] = tf.reshape(tf.concat((flat_frame_probabilities,
tf.reshape(final_probabilities,[-1,1,vocab_size])),axis=1),[-1,vocab_size])
return result
class LstmMultiscaleModel(models.BaseModel):
def cnn(self,
model_input,
l2_penalty=1e-8,
num_filters = [1024, 1024, 1024],
filter_sizes = [1,2,3],
sub_scope="",
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
shift_inputs = []
for i in range(max(filter_sizes)):
if i == 0:
shift_inputs.append(model_input)
else:
shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:])
cnn_outputs = []
for nf, fs in zip(num_filters, filter_sizes):
sub_input = tf.concat(shift_inputs[:fs], axis=2)
sub_filter = tf.get_variable(sub_scope+"cnn-filter-len%d"%fs,
shape=[num_features*fs, nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter))
cnn_output = tf.concat(cnn_outputs, axis=2)
cnn_output = slim.batch_norm(
cnn_output,
center=True,
scale=True,
is_training=FLAGS.train,
scope=sub_scope+"cluster_bn")
return cnn_output, max_frames
def sub_moe(self,model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
scopename="",
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+scopename)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts"+scopename)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
def rnn(self, model_input, lstm_size, num_frames,sub_scope="", **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(1)
],
state_is_tuple=True)
with tf.variable_scope("RNN-"+sub_scope):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32)
state_out = tf.concat(map(lambda x: x.c, state), axis=1)
return state_out
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
num_extend = FLAGS.moe_num_extend
num_layers = num_extend
lstm_size = FLAGS.lstm_cells
pool_size=2
cnn_input = model_input
num_filters=[256,256,512]
filter_sizes=[1,2,3]
features_size = sum(num_filters)
final_probilities = []
moe_inputs = []
for layer in range(num_layers):
cnn_output, num_t = self.cnn(cnn_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn%d"%(layer+1))
cnn_output = tf.nn.relu(cnn_output)
cnn_multiscale = self.rnn(cnn_output,lstm_size, num_frames,sub_scope="rnn%d"%(layer+1))
moe_inputs.append(cnn_multiscale)
final_probility = self.sub_moe(cnn_multiscale,vocab_size,scopename="moe%d"%(layer+1))
final_probilities.append(final_probility)
num_t = pool_size*(num_t//pool_size)
cnn_output = tf.reshape(cnn_output[:,:num_t,:],[-1,num_t//pool_size,pool_size,features_size])
cnn_input = tf.reduce_max(cnn_output, axis=2)
num_frames = tf.maximum(num_frames//pool_size,1)
final_probilities = tf.stack(final_probilities,axis=1)
moe_inputs = tf.stack(moe_inputs,axis=1)
weight2d = tf.get_variable("ensemble_weight2d",
shape=[num_extend, features_size, vocab_size],
regularizer=slim.l2_regularizer(1.0e-8))
weight = tf.nn.softmax(tf.einsum("aij,ijk->aik", moe_inputs, weight2d), dim=1)
result = {}
result["prediction_frames"] = tf.reshape(final_probilities,[-1,vocab_size])
result["predictions"] = tf.reduce_sum(final_probilities*weight,axis=1)
return result
class LstmMultiscaleDitillChainModel(models.BaseModel):
def cnn(self,
model_input,
l2_penalty=1e-8,
num_filters = [1024, 1024, 1024],
filter_sizes = [1,2,3],
sub_scope="",
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
shift_inputs = []
for i in range(max(filter_sizes)):
if i == 0:
shift_inputs.append(model_input)
else:
shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:])
cnn_outputs = []
for nf, fs in zip(num_filters, filter_sizes):
sub_input = tf.concat(shift_inputs[:fs], axis=2)
sub_filter = tf.get_variable(sub_scope+"cnn-filter-len%d"%fs,
shape=[num_features*fs, nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter))
cnn_output = tf.concat(cnn_outputs, axis=2)
cnn_output = slim.batch_norm(
cnn_output,
center=True,
scale=True,
is_training=FLAGS.train,
scope=sub_scope+"cluster_bn")
return cnn_output, max_frames
def sub_moe(self,model_input,
vocab_size,
distill_labels=None,
num_mixtures=None,
l2_penalty=1e-8,
scopename="",
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
class_size = 256
if distill_labels is not None:
class_input = slim.fully_connected(
distill_labels,
class_size,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="class_inputs"+scopename)
model_input = tf.concat((model_input,class_input),axis=1)
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+scopename)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts"+scopename)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
def rnn(self, model_input, lstm_size, num_frames,sub_scope="", **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(1)
],
state_is_tuple=True)
with tf.variable_scope("RNN-"+sub_scope):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32)
state_out = tf.concat(map(lambda x: x.c, state), axis=1)
return state_out
def create_model(self, model_input, vocab_size, num_frames, distill_labels=None, l2_penalty=1e-8, **unused_params):
num_extend = FLAGS.moe_num_extend
num_layers = num_extend
lstm_size = FLAGS.lstm_cells
pool_size = 2
cnn_input = model_input
cnn_size = FLAGS.cnn_cells
num_filters = [cnn_size, cnn_size, cnn_size*2]
filter_sizes = [1, 2, 3]
features_size = sum(num_filters)
final_probilities = []
moe_inputs = []
for layer in range(num_layers):
cnn_output, num_t = self.cnn(cnn_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn%d"%(layer+1))
cnn_output = tf.nn.relu(cnn_output)
cnn_multiscale = self.rnn(cnn_output,lstm_size, num_frames,sub_scope="rnn%d"%(layer+1))
moe_inputs.append(cnn_multiscale)
final_probility = self.sub_moe(cnn_multiscale,vocab_size,distill_labels=distill_labels, scopename="moe%d"%(layer+1))
final_probilities.append(final_probility)
num_t = pool_size*(num_t//pool_size)
cnn_output = tf.reshape(cnn_output[:,:num_t,:],[-1,num_t//pool_size,pool_size,features_size])
cnn_input = tf.reduce_max(cnn_output, axis=2)
num_frames = tf.maximum(num_frames//pool_size,1)
final_probilities = tf.stack(final_probilities,axis=1)
moe_inputs = tf.stack(moe_inputs,axis=1)
weight2d = tf.get_variable("ensemble_weight2d",
shape=[num_extend, lstm_size, vocab_size],
regularizer=slim.l2_regularizer(1.0e-8))
weight = tf.nn.softmax(tf.einsum("aij,ijk->aik", moe_inputs, weight2d), dim=1)
result = {}
result["prediction_frames"] = tf.reshape(final_probilities,[-1,vocab_size])
result["predictions"] = tf.reduce_sum(final_probilities*weight,axis=1)
return result
class LstmMultiscale2Model(models.BaseModel):
def cnn(self,
model_input,
l2_penalty=1e-8,
num_filters=[1024,1024,1024],
filter_sizes=[1,2,3],
sub_scope="",
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
shift_inputs = []
for i in range(max(filter_sizes)):
if i == 0:
shift_inputs.append(model_input)
else:
shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:])
cnn_outputs = []
for nf, fs in zip(num_filters, filter_sizes):
sub_input = tf.concat(shift_inputs[:fs], axis=2)
sub_filter = tf.get_variable(sub_scope+"cnn-filter-len%d"%fs,
shape=[num_features*fs, nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter))
cnn_output = tf.concat(cnn_outputs, axis=2)
cnn_output = slim.batch_norm(
cnn_output,
center=True,
scale=True,
is_training=FLAGS.train,
scope=sub_scope+"cluster_bn")
return cnn_output, max_frames
def sub_moe(self,model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
scopename="",
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+scopename)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts"+scopename)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
def rnn_gate(self, model_input, lstm_size, num_frames, l2_penalty=1e-8, sub_scope="", **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
## Batch normalize the input
max_frames = model_input.get_shape().as_list()[1]
emb_dim = model_input.get_shape().as_list()[2]
hidden_dim = lstm_size
with tf.variable_scope(sub_scope+'lstm_forward'):
g_recurrent_unit_forward = LstmGateModel().create_recurrent_unit(emb_dim,hidden_dim,l2_penalty)
h0 = tf.zeros([tf.shape(model_input)[0], hidden_dim])
h0 = tf.stack([h0, h0])
outputs_state = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
def _pretrain_forward(i,h_tm1,s_predictions):
x_t = model_input[:,i,:]
gate, h_t = g_recurrent_unit_forward(x_t, h_tm1)
hidden_state, c_prev = tf.unstack(h_t)
s_predictions = s_predictions.write(i,c_prev)
return i + 1, h_t, s_predictions
_, _, state_outputs = control_flow_ops.while_loop(
cond=lambda i, _1, _2: i < max_frames,
body=_pretrain_forward,
loop_vars=(tf.constant(0, dtype=tf.int32),h0,outputs_state),
swap_memory=True)
state_outputs = state_outputs.stack()
batch_size = tf.shape(model_input)[0]
index_1 = tf.range(0, batch_size) * max_frames + (num_frames - 1)
state_outputs = tf.transpose(state_outputs, [1, 0, 2])
state_outputs = tf.gather(tf.reshape(state_outputs, [-1, hidden_dim]), index_1)
return state_outputs
def rnn_standard(self, model_input, lstm_size, num_frames,sub_scope="", **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(1)
],
state_is_tuple=True)
with tf.variable_scope("RNN-"+sub_scope):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32)
state_out = tf.concat(map(lambda x: x.c, state), axis=1)
return state_out
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
num_extend = FLAGS.moe_num_extend
num_layers = num_extend
lstm_size = FLAGS.lstm_cells
pool_size = 2
cnn_input = model_input
num_filters = [256, 256, 512]
filter_sizes = [1, 2, 3]
features_size = sum(num_filters)
final_probilities = []
moe_inputs = []
for layer in range(num_layers):
cnn_output, num_t = self.cnn(cnn_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn%d"%(layer+1))
cnn_output = tf.nn.relu(cnn_output)
cnn_multiscale = self.rnn_gate(cnn_output, lstm_size, num_frames, sub_scope="rnn%d"%(layer+1))
moe_inputs.append(cnn_multiscale)
final_probility = self.sub_moe(cnn_multiscale, vocab_size, scopename="moe%d"%(layer+1))
final_probilities.append(final_probility)
num_t = pool_size*(num_t//pool_size)
cnn_output = tf.reshape(cnn_output[:,:num_t,:],[-1,num_t//pool_size,pool_size,features_size])
cnn_input = tf.reduce_max(cnn_output, axis=2)
num_frames = tf.maximum(num_frames//pool_size,1)
final_probilities = tf.stack(final_probilities, axis=1)
moe_inputs = tf.stack(moe_inputs, axis=1)
weight2d = tf.get_variable("ensemble_weight2d",
shape=[num_extend, features_size, vocab_size],
regularizer=slim.l2_regularizer(1.0e-8))
weight = tf.nn.softmax(tf.einsum("aij,ijk->aik", moe_inputs, weight2d), dim=1)
result = {}
result["prediction_frames"] = tf.reshape(final_probilities,[-1, vocab_size])
result["predictions"] = tf.reduce_mean(final_probilities, axis=1)
return result
class LstmMultiscale3Model(models.BaseModel):
def cnn(self,
model_input,
l2_penalty=1e-8,
num_filters=[1024,1024,1024],
filter_sizes=[1,2,3],
sub_scope="",
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
shift_inputs = []
for i in range(max(filter_sizes)):
if i == 0:
shift_inputs.append(model_input)
else:
shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:])
cnn_outputs = []
for nf, fs in zip(num_filters, filter_sizes):
sub_input = tf.concat(shift_inputs[:fs], axis=2)
sub_filter = tf.get_variable(sub_scope+"cnn-filter-len%d"%fs,
shape=[num_features*fs, nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter))
cnn_output = tf.concat(cnn_outputs, axis=2)
cnn_output = slim.batch_norm(
cnn_output,
center=True,
scale=True,
is_training=FLAGS.train,
scope=sub_scope+"cluster_bn")
return cnn_output, max_frames
def sub_moe(self,model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
scopename="",
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+scopename)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts"+scopename)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
def rnn_glu(self, model_input, lstm_size, num_frames, l2_penalty=1e-8, sub_scope="", **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
## Batch normalize the input
max_frames = model_input.get_shape().as_list()[1]
emb_dim = model_input.get_shape().as_list()[2]
hidden_dim = lstm_size
with tf.variable_scope(sub_scope+'lstm_forward'):
g_recurrent_unit_forward = LstmGlu2Model().create_recurrent_unit(emb_dim,hidden_dim,l2_penalty)
h0 = tf.zeros([tf.shape(model_input)[0], hidden_dim])
h0 = tf.stack([h0, h0])
h1 = tf.zeros([tf.shape(model_input)[0], emb_dim])
h1 = tf.stack([h1, h1])
outputs_state = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
def _pretrain_forward(i, h_tm0, h_tm1,s_predictions):
x_t = model_input[:,i,:]
gate, h_t0, h_t1 = g_recurrent_unit_forward(x_t, h_tm0, h_tm1)
hidden_state, c_prev = tf.unstack(h_t1)
s_predictions = s_predictions.write(i,hidden_state)
return i + 1, h_t0, h_t1, s_predictions
_, _, _, state_outputs = control_flow_ops.while_loop(
cond=lambda i, _1, _2, _3: i < max_frames,
body=_pretrain_forward,
loop_vars=(tf.constant(0, dtype=tf.int32), h1, h0, outputs_state),
swap_memory=True)
state_outputs = state_outputs.stack()
batch_size = tf.shape(model_input)[0]
index_1 = tf.range(0, batch_size) * max_frames + (num_frames - 1)
state_outputs = tf.transpose(state_outputs, [1, 0, 2])
state_outputs = tf.gather(tf.reshape(state_outputs, [-1, hidden_dim]), index_1)
return state_outputs
def rnn_standard(self, model_input, lstm_size, num_frames,sub_scope="", **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(1)
],
state_is_tuple=True)
with tf.variable_scope("RNN-"+sub_scope):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32)
state_out = tf.concat(map(lambda x: x.c, state), axis=1)
return state_out
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
num_extend = FLAGS.moe_num_extend
num_layers = num_extend
lstm_size = FLAGS.lstm_cells
pool_size = 2
cnn_input = model_input
num_filters = [256, 256, 512]
filter_sizes = [1, 2, 3]
features_size = sum(num_filters)
final_probilities = []
moe_inputs = []
for layer in range(num_layers):
cnn_output, num_t = self.cnn(cnn_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn%d"%(layer+1))
cnn_output = tf.nn.relu(cnn_output)
cnn_multiscale = self.rnn_glu(cnn_output, lstm_size, num_frames, sub_scope="rnn%d"%(layer+1))
moe_inputs.append(cnn_multiscale)
final_probility = self.sub_moe(cnn_multiscale, vocab_size, scopename="moe%d"%(layer+1))
final_probilities.append(final_probility)
num_t = pool_size*(num_t//pool_size)
cnn_output = tf.reshape(cnn_output[:,:num_t,:],[-1,num_t//pool_size,pool_size,features_size])
cnn_input = tf.reduce_max(cnn_output, axis=2)
num_frames = tf.maximum(num_frames//pool_size,1)
final_probilities = tf.stack(final_probilities, axis=1)
moe_inputs = tf.stack(moe_inputs, axis=1)
weight2d = tf.get_variable("ensemble_weight2d",
shape=[num_extend, features_size, vocab_size],
regularizer=slim.l2_regularizer(1.0e-8))
weight = tf.nn.softmax(tf.einsum("aij,ijk->aik", moe_inputs, weight2d), dim=1)
result = {}
result["prediction_frames"] = tf.reshape(final_probilities,[-1, vocab_size])
result["predictions"] = tf.reduce_mean(final_probilities, axis=1)
return result
class LstmMultiscaleRebuildModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
num_extend = FLAGS.moe_num_extend
num_layers = num_extend
lstm_size = FLAGS.lstm_cells
pool_size=2
cnn_input = model_input
num_filters=[256,256,512]
filter_sizes=[1,2,3]
features_size = sum(num_filters)
final_probilities = []
moe_inputs = []
for layer in range(num_layers):
cnn_output, num_t = LstmMultiscaleModel().cnn(cnn_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn%d"%(layer+1))
cnn_output = tf.nn.relu(cnn_output)
cnn_multiscale = LstmMultiscaleModel().rnn(cnn_output,lstm_size, num_frames,sub_scope="rnn%d"%(layer+1))
moe_inputs.append(cnn_multiscale)
final_probility = LstmMultiscaleModel().sub_moe(cnn_multiscale,vocab_size,scopename="moe%d"%(layer+1))
final_probilities.append(final_probility)
num_t = pool_size*(num_t//pool_size)
cnn_output = tf.reshape(cnn_output[:,:num_t,:],[-1,num_t//pool_size,pool_size,features_size])
cnn_input = tf.reduce_max(cnn_output, axis=2)
num_frames = tf.maximum(num_frames//pool_size,1)
final_probilities = tf.stack(final_probilities, axis=1)
moe_inputs = tf.stack(moe_inputs, axis=1)
weight2d = tf.get_variable("ensemble_weight2d",
shape=[num_extend, features_size, vocab_size],
regularizer=slim.l2_regularizer(1.0e-8))
weight = tf.nn.softmax(tf.einsum("aij,ijk->aik", tf.stop_gradient(moe_inputs), weight2d), dim=1)
result = {}
result["predictions"] = tf.reduce_sum(tf.stop_gradient(final_probilities)*weight, axis=1)
return result
class LstmLayerModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
shape = model_input.get_shape().as_list()
unit_layer_1 = FLAGS.lstm_length
unit_layer_2 = shape[1]//FLAGS.lstm_length
model_input = model_input[:,:unit_layer_1*unit_layer_2,:]
model_input_1 = tf.reshape(model_input,[-1,unit_layer_1,shape[2]])
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(1)
],
state_is_tuple=True)
with tf.variable_scope("RNN-1"):
outputs_1, state_1 = tf.nn.dynamic_rnn(stacked_lstm, model_input_1,
sequence_length=None,
swap_memory=True,
dtype=tf.float32)
state_c = tf.concat(map(lambda x: x.c, state_1), axis=1)
model_input_2 = tf.reshape(state_c,[-1,unit_layer_2,lstm_size])
num_frames = num_frames//unit_layer_1
with tf.variable_scope("RNN-2"):
outputs_2, state_2 = tf.nn.dynamic_rnn(stacked_lstm, model_input_2,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32)
state_out = tf.concat(map(lambda x: x.c, state_2), axis=1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=state_out,
vocab_size=vocab_size,
**unused_params)
class LstmDivideModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
num_extend = FLAGS.moe_num_extend - 1
shape = model_input.get_shape().as_list()
full_num_frames = num_frames
num_frames = tf.maximum(num_frames//num_extend,1)
length = shape[1]//num_extend
divide_inputs = []
divide_inputs.append(model_input)
for i in range(num_extend):
begin_frames = tf.reshape(num_frames*i,[-1,1])
frames_index = tf.reshape(tf.range(length),[1,length])
frames_index = begin_frames+frames_index
batch_size = tf.shape(model_input)[0]
batch_index = tf.tile(
tf.expand_dims(tf.range(batch_size), 1), [1, length])
index = tf.stack([batch_index, tf.cast(frames_index,dtype=tf.int32)], 2)
divide_input = tf.gather_nd(model_input, index)
divide_input = tf.pad(divide_input, paddings=[[0,0],[0,shape[1]-length],[0,0]])
divide_inputs.append(divide_input)
divide_inputs = tf.reshape(tf.stack(divide_inputs,axis=1),[-1,shape[1],shape[2]])
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
num_frames = tf.tile(tf.reshape(num_frames,[-1,1]),[1,num_extend])
num_frames = tf.concat((tf.reshape(full_num_frames,[-1,1]),num_frames),axis=1)
num_frames = tf.reshape(num_frames,[-1])
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, divide_inputs,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32)
state_c = tf.concat(map(lambda x: x.c, state), axis=1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
result = aggregated_model().create_model(
model_input=state_c,
vocab_size=vocab_size,
**unused_params)
final_probilities = result["predictions"]
result["prediction_frames"] = final_probilities
result["predictions"] = tf.reduce_mean(tf.reshape(final_probilities,[-1,num_extend+1,vocab_size]),axis=1)
return result
class LstmDivideRebuildModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32)
state_c = tf.concat(map(lambda x: x.c, state), axis=1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
result = aggregated_model().create_model(
model_input=state_c,
vocab_size=vocab_size,
**unused_params)
return result
class LstmDivideRebuild2Model(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
num_extend = FLAGS.moe_num_extend - 1
shape = model_input.get_shape().as_list()
full_num_frames = num_frames
num_frames = tf.maximum(num_frames//num_extend,1)
length = shape[1]//num_extend
divide_inputs = []
divide_inputs.append(model_input)
for i in range(num_extend):
begin_frames = tf.reshape(num_frames*i,[-1,1])
frames_index = tf.reshape(tf.range(length),[1,length])
frames_index = begin_frames+frames_index
batch_size = tf.shape(model_input)[0]
batch_index = tf.tile(
tf.expand_dims(tf.range(batch_size), 1), [1, length])
index = tf.stack([batch_index, tf.cast(frames_index,dtype=tf.int32)], 2)
divide_input = tf.gather_nd(model_input, index)
divide_input = tf.pad(divide_input, paddings=[[0,0],[0,shape[1]-length],[0,0]])
divide_inputs.append(divide_input)
divide_inputs = tf.reshape(tf.stack(divide_inputs,axis=1),[-1,shape[1],shape[2]])
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
num_frames = tf.tile(tf.reshape(num_frames,[-1,1]),[1,num_extend])
num_frames = tf.concat((tf.reshape(full_num_frames,[-1,1]),num_frames),axis=1)
num_frames = tf.reshape(num_frames,[-1])
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, divide_inputs,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32)
state_c = tf.concat(map(lambda x: x.c, state), axis=1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
result = aggregated_model().create_model(
model_input=state_c,
vocab_size=vocab_size,
**unused_params)
features_size = state_c.get_shape().as_list()[1]
final_probilities = result["predictions"]
final_probilities_divide = tf.reshape(final_probilities,[-1,num_extend,vocab_size])
moe_inputs = tf.reshape(state_c,[-1, num_extend, features_size])
weight2d = tf.get_variable("notrestore_ensemble_weight2d",
shape=[features_size, vocab_size],
regularizer=slim.l2_regularizer(1.0e-8))
weight = tf.nn.softmax(tf.einsum("aij,jk->aik", tf.stop_gradient(moe_inputs), weight2d), dim=1)
result = {}
result["predictions"] = tf.reduce_sum(tf.stop_gradient(final_probilities_divide)*weight,axis=1)
return result
class LstmResidualModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
shortcut_interval = FLAGS.lstm_interval
shape = model_input.get_shape().as_list()
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
with tf.variable_scope("RNN"):
outputs, state = rnn_residual.dynamic_rnn(stacked_lstm, model_input,shortcut_interval,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32)
state_c = tf.concat(map(lambda x: x.c, state), axis=1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=state_c,
vocab_size=vocab_size,
**unused_params)
class LstmNoiseModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
if FLAGS.train=="train":
noise = tf.random_normal(shape=tf.shape(model_input), mean=0.0, stddev=FLAGS.noise_std, dtype=tf.float32)
model_input = tf.nn.l2_normalize(model_input+noise, 2)
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32)
state_c = tf.concat(map(lambda x: x.c, state), axis=1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=state_c,
vocab_size=vocab_size,
**unused_params)
class LstmConditionModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
shape = model_input.get_shape().as_list()
batch_size = tf.shape(model_input)[0]
condition_input = tf.tile(tf.reshape(tf.diag(tf.ones(shape[1])),[1,shape[1],shape[1]]),[batch_size,1,1])
model_input = tf.concat((model_input,condition_input),axis=2)
model_input = tf.nn.l2_normalize(model_input, 2)
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32)
state_c = tf.concat(map(lambda x: x.c, state), axis=1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=state_c,
vocab_size=vocab_size,
**unused_params)
class LstmRandomModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
if FLAGS.train=="train":
shape = model_input.get_shape().as_list()
frames_sum = tf.reduce_sum(tf.abs(model_input),axis=2)
frames_index = tf.reshape(tf.range(shape[1],dtype=tf.float32),[1,shape[1]])
frames_index = tf.ones(tf.shape(frames_sum))*frames_index
frames_index = tf.transpose(tf.random_shuffle(tf.transpose(frames_index)))
frames_index = frames_index[:,0:shape[1]//2]
frames_index = tf.negative(frames_index)
frames_index, indexes = tf.nn.top_k(frames_index, sorted=True, k=shape[1]//2)
frames_index = tf.negative(frames_index)
frames_valid = tf.cast(tf.reshape(num_frames,[-1,1]),dtype=tf.float32)
frames_true = tf.ones(tf.shape(frames_index))
frames_false = tf.zeros(tf.shape(frames_index))
frames_bool = tf.where(tf.less(frames_index, frames_valid), frames_true, frames_false)
num_frames = tf.reduce_sum(frames_bool,axis=1)
batch_size = tf.shape(model_input)[0]
batch_index = tf.tile(
tf.expand_dims(tf.range(batch_size), 1), [1, shape[1]//2])
index = tf.stack([batch_index, tf.cast(frames_index,dtype=tf.int32)], 2)
model_input = tf.gather_nd(model_input, index)
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32)
state_c = tf.concat(map(lambda x: x.c, state), axis=1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=state_c,
vocab_size=vocab_size,
**unused_params)
class LstmAttentionModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames,l2_penalty=1e-8, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
shape = model_input.get_shape().as_list()
frames_sum = tf.reduce_sum(tf.abs(model_input),axis=2)
frames_true = tf.ones(tf.shape(frames_sum))
frames_false = tf.zeros(tf.shape(frames_sum))
frames_bool = tf.where(tf.greater(frames_sum, frames_false), frames_true, frames_false)
frames_bool = tf.reshape(frames_bool,[-1,shape[1]])
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32)
state_c = tf.concat(map(lambda x: x.c, state), axis=1)
state_h = tf.concat(map(lambda x: x.h, state), axis=1)
state_all = tf.concat((state_c, state_h), axis=1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
vocab_input = state_c
attention_predictions = []
class_size = FLAGS.class_size
channels = vocab_size//class_size + 1
hidden_outputs = tf.reshape(outputs,[-1,lstm_size])
k = tf.get_variable("AttnW", [lstm_size,lstm_size], tf.float32,
initializer=tf.contrib.layers.xavier_initializer())
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(k))
hidden_features = tf.matmul(hidden_outputs, k)
hidden_features = tf.reshape(hidden_features,[-1,shape[1],lstm_size])
#v = tf.get_variable("AttnV", [lstm_size,1], tf.float32,initializer=tf.contrib.layers.xavier_initializer())
#tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(v))
state_w = tf.get_variable("state_w", [lstm_size*2,lstm_size], tf.float32,
initializer=tf.contrib.layers.xavier_initializer())
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(state_w))
state_v = tf.get_variable("state_v" , [lstm_size], tf.float32,
initializer=tf.constant_initializer(0.0))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(state_v))
def attention(query):
"""Put attention masks on hidden using hidden_features and query."""
y = tf.reshape(query, [-1,lstm_size,1])
# Attention mask is a softmax of v^T * tanh(...).
s = tf.reshape(tf.einsum('aij,ajk->aik', hidden_features, y), [-1,shape[1]])
a = tf.nn.softmax(s)
rnn_attention = a*frames_bool
rnn_attention = rnn_attention/(tf.reduce_sum(rnn_attention, axis=1, keep_dims=True)+1e-6)
# Now calculate the attention-weighted vector d.
d = tf.einsum('aij,ajk->aik', tf.reshape(rnn_attention, [-1, 1, shape[1]]), outputs)
ds = tf.reshape(d, [-1, lstm_size])
return ds
for i in range(channels):
if iijl", sub_input, sub_filter))
sigmoid_filter = tf.get_variable(sub_scope+"sigmoid-filter-len%d"%fs,
shape=[num_features*fs, nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_sigmoids.append(tf.nn.sigmoid(tf.einsum("ijk,kl->ijl", sub_input, sigmoid_filter)))
cnn_output = tf.concat(cnn_outputs, axis=2)
cnn_sigmoid = tf.concat(cnn_sigmoids, axis=2)
if cnn_output.get_shape().as_list()[2]==model_input.get_shape().as_list()[2]:
cnn_output = cnn_output*cnn_sigmoid + model_input
else:
cnn_output = cnn_output*cnn_sigmoid
cnn_output = slim.batch_norm(
cnn_output,
center=True,
scale=True,
is_training=FLAGS.train,
scope=sub_scope+"cluster_bn")
return cnn_output, max_frames
def kmax(self,
model_input,
l2_penalty=1e-8,
num_filter=1024,
filter_size=8,
sub_scope="",
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
num_filters = [num_filter for _ in range(filter_size)]
filter_sizes = [i*2+1 for i in range(filter_size)]
shift_inputs = []
for i in range(max(filter_sizes)):
if i == 0:
shift_inputs.append(model_input)
else:
shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:])
cnn_outputs = []
for nf, fs in zip(num_filters, filter_sizes):
sub_input = tf.concat(shift_inputs[:fs], axis=2)
sub_filter = tf.get_variable(sub_scope+"cnn-filter-len%d"%fs,
shape=[num_features*fs, nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter))
cnn_output = tf.stack(cnn_outputs, axis=2)
cnn_output = tf.reduce_max(cnn_output, axis=1)
return cnn_output, max_frames
def sub_moe(self,
model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
scopename="",
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+scopename)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts"+scopename)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
num_extend = FLAGS.moe_num_extend
num_layers = 10
pool_size=2
cnn_input = model_input
num_filters=[256,256,512]
filter_sizes=[1,2,3]
features_size = sum(num_filters)
for layer in range(num_layers):
cnn_output, num_t = self.cnn(cnn_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn%d"%(layer+1))
if layer < 3:
num_t = pool_size*(num_t//pool_size)
cnn_output = tf.reshape(cnn_output[:,:num_t,:],[-1,num_t//pool_size,pool_size,features_size])
cnn_input = tf.reduce_max(cnn_output, axis=2)
else:
cnn_input = cnn_output
cnn_output, num_t = self.kmax(cnn_input, num_filters=features_size, filter_sizes=num_extend, sub_scope="kmax")
cnn_input = tf.reshape(cnn_output,[-1,features_size])
final_probilities = self.sub_moe(cnn_input,vocab_size)
final_probilities = tf.reshape(final_probilities,[-1,num_extend,vocab_size])
weight2d = tf.get_variable("ensemble_weight2d",
shape=[num_extend, features_size, vocab_size],
regularizer=slim.l2_regularizer(1.0e-8))
weight = tf.nn.softmax(tf.einsum("aij,ijk->aik", cnn_output, weight2d), dim=1)
result = {}
result["predictions"] = tf.reduce_sum(final_probilities*weight,axis=1)
return result
class CnnLstmModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
# Create a convolution + maxpool layer for each filter size
filter_sizes = [1, 2, 3]
num_frames = num_frames-filter_sizes[-1]+1
shape = model_input.get_shape().as_list()
slice = shape[1]-filter_sizes[-1]+1
num_filters = [384,384,384]
pooled_outputs = []
frames_sum = tf.reduce_sum(tf.abs(model_input),axis=2)
frames_true = tf.ones(tf.shape(frames_sum))
frames_false = tf.zeros(tf.shape(frames_sum))
frames_bool = tf.where(tf.greater(frames_sum, frames_false), frames_true, frames_false)
frames_bool = tf.reshape(frames_bool,[-1,shape[1],1])
frames_bool = frames_bool/tf.reduce_sum(frames_bool,axis=1,keep_dims=True)
with tf.variable_scope("CNN"):
cnn_input = tf.expand_dims(model_input, 3)
for filter_size, num_filter in zip(filter_sizes, num_filters):
with tf.name_scope("conv-maxpool-%s" % filter_size):
# Convolution Layer
#paddings = tf.constant([[0,0],[filter_size//2,filter_size-filter_size//2-1],[0,0]])
filter_shape = [filter_size, shape[2], 1, num_filter]
W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name="W")
b = tf.Variable(tf.constant(0.1, shape=[num_filter]), name="b")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b))
conv = tf.nn.conv2d(cnn_input,W,strides=[1, 1, 1, 1],padding="VALID",name="conv")
# Apply nonlinearity
h = tf.nn.relu(tf.nn.bias_add(conv, b), name="relu")
# Maxpooling over the outputs
h_shape = h.get_shape().as_list()
h_out = tf.reshape(h,[-1,h_shape[1],num_filter])
#h_out = tf.pad(h_out,paddings)
h_out = h_out[:,0:slice,:]
pooled_outputs.append(h_out)
# Combine all the pooled features
h_pool = tf.concat(pooled_outputs,2)
lstm_size = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=False)
for _ in range(number_of_layers)
],
state_is_tuple=False)
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, h_pool,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32)
output = tf.reduce_sum(outputs*frames_bool[:,0:slice,:],axis=1)
#output = tf.reduce_max(outputs,axis=1)
pooled = tf.concat((output,state),axis=1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=pooled,
vocab_size=vocab_size,
**unused_params)
class CnnKmaxModel(models.BaseModel):
def cnn(self,
model_input,
l2_penalty=1e-8,
num_filters = [1024, 1024, 1024],
filter_sizes = [1,2,3],
sub_scope="",
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
shift_inputs = []
for i in range(max(filter_sizes)):
if i == 0:
shift_inputs.append(model_input)
else:
shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:])
cnn_outputs = []
for nf, fs in zip(num_filters, filter_sizes):
sub_input = tf.concat(shift_inputs[:fs], axis=2)
sub_filter = tf.get_variable(sub_scope+"cnn-filter-len%d"%fs,
shape=[num_features*fs, nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter))
cnn_output = tf.concat(cnn_outputs, axis=2)
cnn_output = slim.batch_norm(
cnn_output,
center=True,
scale=True,
is_training=FLAGS.train,
scope=sub_scope+"cluster_bn")
return cnn_output, max_frames
def sub_moe(self,
model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
scopename="",
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+scopename)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts"+scopename)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
num_extend = FLAGS.moe_num_extend
num_layers = num_extend
pool_size=2
cnn_input = model_input
num_filters=[256,256,512]
filter_sizes=[1,2,3]
features_size = sum(num_filters)
final_probilities = []
moe_inputs = []
for layer in range(num_layers):
cnn_output, num_t = self.cnn(cnn_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn%d"%(layer+1))
cnn_output = tf.nn.relu(cnn_output)
cnn_multiscale = tf.reduce_max(cnn_output,axis=1)
moe_inputs.append(cnn_multiscale)
final_probility = self.sub_moe(cnn_multiscale,vocab_size,scopename="moe%d"%(layer+1))
final_probilities.append(final_probility)
num_t = pool_size*(num_t//pool_size)
cnn_output = tf.reshape(cnn_output[:,:num_t,:],[-1,num_t//pool_size,pool_size,features_size])
cnn_input = tf.reduce_max(cnn_output, axis=2)
final_probilities = tf.stack(final_probilities,axis=1)
moe_inputs = tf.stack(moe_inputs,axis=1)
weight2d = tf.get_variable("ensemble_weight2d",
shape=[num_extend, features_size, vocab_size],
regularizer=slim.l2_regularizer(1.0e-8))
weight = tf.nn.softmax(tf.einsum("aij,ijk->aik", moe_inputs, weight2d), dim=1)
result = {}
result["prediction_frames"] = tf.reshape(final_probilities,[-1,vocab_size])
result["predictions"] = tf.reduce_sum(final_probilities*weight,axis=1)
return result
class CnnKmaxRebuildModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
num_extend = FLAGS.moe_num_extend
num_layers = num_extend
pool_size=2
cnn_input = model_input
num_filters=[256,256,512]
filter_sizes=[1,2,3]
features_size = sum(num_filters)
final_probilities = []
moe_inputs = []
for layer in range(num_layers):
cnn_output, num_t = CnnKmaxModel().cnn(cnn_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn%d"%(layer+1), l2_penalty=0.0)
cnn_output = tf.nn.relu(cnn_output)
cnn_multiscale = tf.reduce_max(cnn_output,axis=1)
moe_inputs.append(cnn_multiscale)
final_probility = CnnKmaxModel().sub_moe(cnn_multiscale,vocab_size,scopename="moe%d"%(layer+1), l2_penalty=0.0)
final_probilities.append(final_probility)
num_t = pool_size*(num_t//pool_size)
cnn_output = tf.reshape(cnn_output[:,:num_t,:],[-1,num_t//pool_size,pool_size,features_size])
cnn_input = tf.reduce_max(cnn_output, axis=2)
final_probilities = tf.stack(final_probilities,axis=1)
moe_inputs = tf.stack(moe_inputs,axis=1)
weight2d = tf.get_variable("ensemble_weight2d",
shape=[num_extend, features_size, vocab_size],
regularizer=slim.l2_regularizer(1.0e-8))
weight = tf.nn.softmax(tf.einsum("aij,ijk->aik", tf.stop_gradient(moe_inputs), weight2d), dim=1)
result = {}
result["predictions"] = tf.reduce_sum(tf.stop_gradient(final_probilities)*weight, axis=1)
return result
class CnnWholeModel(models.BaseModel):
def cnn(self,
model_input,
l2_penalty=1e-8,
num_filters=[4,512],
sub_scope="",
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
sub_filter_1 = tf.get_variable(sub_scope+"cnn-filter-1",
shape=[max_frames, num_filters[0]], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
sub_bias_1 = tf.get_variable(sub_scope+"cnn-bias-1",
shape=[num_filters[0]], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs_1 = tf.einsum("ijk,jl->ikl", model_input, sub_filter_1) + sub_bias_1
cnn_outputs_1 = slim.batch_norm(
cnn_outputs_1,
center=True,
scale=True,
is_training=FLAGS.train,
scope=sub_scope+"cluster_bn_1")
cnn_outputs_1 = tf.nn.relu(cnn_outputs_1)
sub_filter_2 = tf.get_variable(sub_scope+"cnn-filter-2",
shape=[num_features, num_filters[1]], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
sub_bias_2 = tf.get_variable(sub_scope+"cnn-bias-2",
shape=[num_filters[1]], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs_2 = tf.einsum("ijk,jl->ikl", cnn_outputs_1, sub_filter_2) + sub_bias_2
cnn_outputs_2 = tf.reshape(cnn_outputs_2,[-1,num_filters[0]*num_filters[1]])
cnn_output = slim.batch_norm(
cnn_outputs_2,
center=True,
scale=True,
is_training=FLAGS.train,
scope=sub_scope+"cluster_bn_2")
return cnn_output
def sub_moe(self,
model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
scopename="",
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+scopename)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts"+scopename)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
cnn_output = self.cnn(model_input, sub_scope="cnn")
final_probility = self.sub_moe(cnn_output,vocab_size,scopename="moe")
result = {}
result["predictions"] = final_probility
return result
class CnnMultiscaleModel(models.BaseModel):
def cnn(self,
model_input,
l2_penalty=1e-8,
num_filters = [1024, 1024, 1024],
filter_sizes = [1,2,3],
sub_scope="",
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
shift_inputs = []
for i in range(max(filter_sizes)):
if i == 0:
shift_inputs.append(model_input)
else:
shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:])
cnn_outputs = []
for nf, fs in zip(num_filters, filter_sizes):
sub_input = tf.concat(shift_inputs[:fs], axis=2)
sub_filter = tf.get_variable(sub_scope+"cnn-filter-len%d"%fs,
shape=[num_features*fs, nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
sub_bias = tf.get_variable(sub_scope+"cnn-bias-len%d"%fs,
shape=[nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter) + sub_bias)
cnn_output = tf.concat(cnn_outputs, axis=2)
cnn_output = slim.batch_norm(
cnn_output,
center=True,
scale=True,
is_training=FLAGS.train,
scope=sub_scope+"cluster_bn")
return cnn_output, max_frames
def sub_moe(self,
model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
scopename="",
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+scopename)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts"+scopename)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
num_extend = FLAGS.moe_num_extend
num_layers = num_extend
pool_size=2
cnn_input = model_input
num_filters=[256,256,512]
filter_sizes=[1,2,3]
features_size = sum(num_filters)
final_probilities = []
moe_inputs = []
for layer in range(num_layers):
cnn_output, num_t = self.cnn(cnn_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn%d"%(layer+1))
cnn_output = tf.nn.relu(cnn_output)
cnn_multiscale = tf.reduce_max(cnn_output,axis=1)
moe_inputs.append(cnn_multiscale)
final_probility = self.sub_moe(cnn_multiscale,vocab_size,scopename="moe%d"%(layer+1))
final_probilities.append(final_probility)
num_t = pool_size*(num_t//pool_size)
cnn_output = tf.reshape(cnn_output[:,:num_t,:],[-1,num_t//pool_size,pool_size,features_size])
cnn_input = tf.reduce_max(cnn_output, axis=2)
final_probilities = tf.stack(final_probilities,axis=1)
moe_inputs = tf.stack(moe_inputs,axis=1)
weight2d = tf.get_variable("ensemble_weight2d",
shape=[num_extend, features_size, vocab_size],
regularizer=slim.l2_regularizer(1.0e-8))
weight = tf.nn.softmax(tf.einsum("aij,ijk->aik", moe_inputs, weight2d), dim=1)
result = {}
result["prediction_frames"] = tf.reshape(final_probilities,[-1,vocab_size])
result["predictions"] = tf.reduce_sum(final_probilities*weight,axis=1)
return result
class DeepCnnModel(models.BaseModel):
def highway(self, input_1, input_2, size_1, size_2, l2_penalty=1e-8, layer_size=1):
output = input_2
for idx in range(layer_size):
with tf.name_scope('output_lin_%d' % idx):
W = tf.Variable(tf.truncated_normal([size_2,size_1], stddev=0.1), name="W")
b = tf.Variable(tf.constant(0.1, shape=[size_1]), name="b")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b))
output = tf.nn.relu(tf.nn.xw_plus_b(output,W,b))
with tf.name_scope('transform_lin_%d' % idx):
W = tf.Variable(tf.truncated_normal([size_1,size_1], stddev=0.1), name="W")
b = tf.Variable(tf.constant(0.1, shape=[size_1]), name="b")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b))
transform_gate = tf.sigmoid(tf.nn.xw_plus_b(input_1,W,b))
carry_gate = tf.constant(1.0) - transform_gate
output = transform_gate * output + carry_gate * input_1
return output
def conv_block(self, input, out_size, layer, kernalsize=3, l2_penalty=1e-8, shortcut=False):
in_shape = input.get_shape().as_list()
if layer>0:
filter_shape = [kernalsize, 1, in_shape[3], out_size]
else:
filter_shape = [kernalsize, in_shape[2], 1, out_size]
W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name="W-%s" % layer)
b = tf.Variable(tf.constant(0.1, shape=[out_size]), name="b-%s" % layer)
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b))
if layer>0:
conv = tf.nn.conv2d(input, W, strides=[1, 1, 1, 1], padding="SAME", name="conv-%s" % layer)
else:
conv = tf.nn.conv2d(input, W, strides=[1, 1, 1, 1], padding="VALID", name="conv-%s" % layer)
if shortcut:
shortshape = [1,1,in_shape[3], out_size]
Ws = tf.Variable(tf.truncated_normal(shortshape, stddev=0.05), name="Ws-%s" % layer)
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Ws))
conv = conv + tf.nn.conv2d(input, Ws, strides=[1, 1, 1, 1], padding="SAME", name="conv-shortcut-%s" % layer)
h = tf.nn.bias_add(conv, b)
h2 = tf.nn.relu(tf.contrib.layers.batch_norm(h, center=True, scale=True, epsilon=1e-5, decay=0.9), name="relu-%s" % layer)
return h2
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
# Create a convolution + maxpool layer for each filter size
kernal_size=3
pool_size=2
num_extend = FLAGS.moe_num_extend
num_filters = 1024
layers = 4
with tf.variable_scope("CNN"):
cnn_input = tf.expand_dims(model_input, 3)
cnn_output = []
for i in range(layers):
with tf.name_scope("conv-maxpool-%s" % i):
# Convolution Layer
cnn_input = self.conv_block(cnn_input,l2_penalty=l2_penalty, out_size=num_filters, layer=i, kernalsize=kernal_size)
if iaik', rnn_attention, outputs)
pooled = tf.reshape(rnn_out,[-1,lstm_size])
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
result = aggregated_model().create_model(
model_input=pooled,
vocab_size=vocab_size,
**unused_params)
result["attention_weight"] = tf.reshape(rnn_attention,[-1, shape[1]*num_extend])
return result
class LstmExtendStateModel(models.BaseModel):
def cnn(self,
model_input,
l2_penalty=1e-8,
num_filters=[1024, 1024, 1024],
filter_sizes=[1,2,3],
sub_scope="",
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
shift_inputs = []
for i in range(max(filter_sizes)):
if i == 0:
shift_inputs.append(model_input)
else:
shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:])
cnn_outputs = []
for nf, fs in zip(num_filters, filter_sizes):
sub_input = tf.concat(shift_inputs[:fs], axis=2)
sub_filter = tf.get_variable(sub_scope+"cnn-filter-len%d"%fs,
shape=[num_features*fs, nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter))
cnn_output = tf.concat(cnn_outputs, axis=2)
return cnn_output, max_frames
def create_model(self, model_input, vocab_size, num_frames,l2_penalty=1e-8, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
num_extend = FLAGS.moe_num_extend
shape = model_input.get_shape().as_list()
frames_sum = tf.reduce_sum(tf.abs(model_input),axis=2)
frames_true = tf.ones(tf.shape(frames_sum))
frames_false = tf.zeros(tf.shape(frames_sum))
frames_bool = tf.where(tf.greater(frames_sum, frames_false), frames_true, frames_false)
frames_bool = tf.reshape(frames_bool,[-1,shape[1],1])
num_filters=[256,256,512]
filter_sizes = [1,2,3]
cnn_size = sum(num_filters)
cnn_output, _ = self.cnn(model_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn")
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32)
final_output = tf.concat(map(lambda x: x.c, state), axis=1)
final_output = tf.tile(tf.reshape(final_output,[-1,1,lstm_size*2]),[1,shape[1],1])
W1 = tf.Variable(tf.truncated_normal([lstm_size*2,num_extend], stddev=0.1), name="W1")
b1 = tf.Variable(tf.constant(0.1, shape=[num_extend]), name="b1")
W2 = tf.Variable(tf.truncated_normal([cnn_size,num_extend], stddev=0.1), name="W2")
b2 = tf.Variable(tf.constant(0.1, shape=[num_extend]), name="b2")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W1))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b1))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W2))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b2))
rnn_output = tf.reshape(final_output,[-1,lstm_size*2])
rnn_input = tf.reshape(cnn_output,[-1,cnn_size])
rnn_attention = tf.nn.softmax(tf.reshape(tf.nn.xw_plus_b(rnn_output,W1,b1)+tf.nn.xw_plus_b(rnn_input,W2,b2),[-1,shape[1],num_extend]),dim=1)*frames_bool
rnn_attention = rnn_attention/tf.reduce_sum(rnn_attention, axis=1, keep_dims=True)
rnn_attention = tf.transpose(rnn_attention,perm=[0,2,1])
rnn_out = tf.einsum('aij,ajk->aik', rnn_attention, cnn_output)
pooled = tf.reshape(rnn_out,[-1,cnn_size])
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
result = aggregated_model().create_model(
model_input=pooled,
vocab_size=vocab_size,
**unused_params)
result["attention_weight"] = tf.reshape(rnn_attention,[-1, shape[1]*num_extend])
return result
class LstmExtendInputModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames,l2_penalty=1e-8, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
num_extend = FLAGS.moe_num_extend
shape = model_input.get_shape().as_list()
frames_sum = tf.reduce_sum(tf.abs(model_input),axis=2)
frames_true = tf.ones(tf.shape(frames_sum))
frames_false = tf.zeros(tf.shape(frames_sum))
frames_bool = tf.where(tf.greater(frames_sum, frames_false), frames_true, frames_false)
frames_bool = tf.reshape(frames_bool,[-1,shape[1],1])
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=False)
for _ in range(number_of_layers)
],
state_is_tuple=False)
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32)
W1 = tf.Variable(tf.truncated_normal([lstm_size,num_extend], stddev=0.1), name="W1")
b1 = tf.Variable(tf.constant(0.1, shape=[num_extend]), name="b1")
W2 = tf.Variable(tf.truncated_normal([shape[2],num_extend], stddev=0.1), name="W2")
b2 = tf.Variable(tf.constant(0.1, shape=[num_extend]), name="b2")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W1))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b1))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W2))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b2))
rnn_output = tf.reshape(outputs,[-1,lstm_size])
rnn_input = tf.reshape(model_input,[-1,shape[2]])
rnn_attention = tf.nn.softmax(tf.reshape(tf.nn.xw_plus_b(rnn_output,W1,b1)+tf.nn.xw_plus_b(rnn_input,W2,b2),[-1,shape[1],num_extend]),dim=1)*frames_bool
rnn_attention = rnn_attention/tf.reduce_sum(rnn_attention, axis=1, keep_dims=True)
rnn_attention = tf.transpose(rnn_attention,perm=[0,2,1])
rnn_out = tf.einsum('aij,ajk->aik', rnn_attention, model_input)
pooled = tf.reshape(rnn_out,[-1,shape[2]])
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
result = aggregated_model().create_model(
model_input=pooled,
vocab_size=vocab_size,
**unused_params)
result["attention_weight"] = tf.reshape(rnn_attention,[-1, shape[1]*num_extend])
return result
class LstmExtendCNNModel(models.BaseModel):
def cnn(self,
model_input,
l2_penalty=1e-8,
num_filters=[1024, 1024, 1024],
filter_sizes=[1,2,3],
sub_scope="",
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
shift_inputs = []
for i in range(max(filter_sizes)):
if i == 0:
shift_inputs.append(model_input)
else:
shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:])
cnn_outputs = []
for nf, fs in zip(num_filters, filter_sizes):
sub_input = tf.concat(shift_inputs[:fs], axis=2)
sub_filter = tf.get_variable(sub_scope+"cnn-filter-len%d"%fs,
shape=[num_features*fs, nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter))
cnn_output = tf.concat(cnn_outputs, axis=2)
return cnn_output, max_frames
def create_model(self, model_input, vocab_size, num_frames,l2_penalty=1e-8, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
num_extend = FLAGS.moe_num_extend
shape = model_input.get_shape().as_list()
frames_sum = tf.reduce_sum(tf.abs(model_input),axis=2)
frames_true = tf.ones(tf.shape(frames_sum))
frames_false = tf.zeros(tf.shape(frames_sum))
frames_bool = tf.where(tf.greater(frames_sum, frames_false), frames_true, frames_false)
frames_bool = tf.reshape(frames_bool,[-1,shape[1],1])
num_filters=[256,256,512]
filter_sizes = [1,2,3]
cnn_size = sum(num_filters)
cnn_output, _ = self.cnn(model_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn")
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=False)
for _ in range(number_of_layers)
],
state_is_tuple=False)
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32)
W1 = tf.Variable(tf.truncated_normal([lstm_size,num_extend], stddev=0.1), name="W1")
b1 = tf.Variable(tf.constant(0.1, shape=[num_extend]), name="b1")
W2 = tf.Variable(tf.truncated_normal([cnn_size,num_extend], stddev=0.1), name="W2")
b2 = tf.Variable(tf.constant(0.1, shape=[num_extend]), name="b2")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W1))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b1))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W2))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b2))
rnn_output = tf.reshape(outputs,[-1,lstm_size])
rnn_input = tf.reshape(cnn_output,[-1,cnn_size])
rnn_attention = tf.nn.softmax(tf.reshape(tf.nn.xw_plus_b(rnn_output,W1,b1)+tf.nn.xw_plus_b(rnn_input,W2,b2),[-1,shape[1],num_extend]),dim=1)*frames_bool
rnn_attention = rnn_attention/tf.reduce_sum(rnn_attention, axis=1, keep_dims=True)
rnn_attention = tf.transpose(rnn_attention,perm=[0,2,1])
rnn_out = tf.einsum('aij,ajk->aik', rnn_attention, cnn_output)
pooled = tf.reshape(rnn_out,[-1,cnn_size])
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
result = aggregated_model().create_model(
model_input=pooled,
vocab_size=vocab_size,
**unused_params)
result["attention_weight"] = tf.reshape(rnn_attention,[-1, shape[1]*num_extend])
return result
class LstmExtendOrthoModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames,l2_penalty=1e-8, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
num_extend = FLAGS.moe_num_extend
shape = model_input.get_shape().as_list()
frames_sum = tf.reduce_sum(tf.abs(model_input),axis=2)
frames_true = tf.ones(tf.shape(frames_sum))
frames_false = tf.zeros(tf.shape(frames_sum))
frames_bool = tf.where(tf.greater(frames_sum, frames_false), frames_true, frames_false)
frames_bool = tf.reshape(frames_bool,[-1,shape[1],1])
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=False)
for _ in range(number_of_layers)
],
state_is_tuple=False)
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32)
def calculate_ortho(W):
index = tf.cast(tf.random_uniform([2])*num_extend, tf.int32)
vec_1 = W[:, index[0]]
vec_2 = W[:, index[1]]
ortho = tf.square(tf.reduce_sum(vec_1*vec_2))/(tf.reduce_sum(tf.square(vec_1))*tf.reduce_sum(tf.square(vec_2)))
return ortho
W1 = tf.Variable(tf.truncated_normal([lstm_size,num_extend], stddev=0.1), name="W1")
b1 = tf.Variable(tf.constant(0.1, shape=[num_extend]), name="b1")
W2 = tf.Variable(tf.truncated_normal([shape[2],num_extend], stddev=0.1), name="W2")
b2 = tf.Variable(tf.constant(0.1, shape=[num_extend]), name="b2")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W1)+calculate_ortho(W1))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b1))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W2)+calculate_ortho(W1))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b2))
rnn_output = tf.reshape(outputs,[-1,lstm_size])
rnn_input = tf.reshape(model_input,[-1,shape[2]])
rnn_attention = tf.nn.softmax(tf.reshape(tf.nn.xw_plus_b(rnn_output,W1,b1)+tf.nn.xw_plus_b(rnn_input,W2,b2),[-1,shape[1],num_extend]),dim=1)*frames_bool
rnn_attention = rnn_attention/tf.reduce_sum(rnn_attention, axis=1, keep_dims=True)
rnn_attention = tf.transpose(rnn_attention,perm=[0,2,1])
rnn_out = tf.einsum('aij,ajk->aik', rnn_attention, outputs)
pooled = tf.reshape(rnn_out,[-1,lstm_size])
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=pooled,
vocab_size=vocab_size,
**unused_params)
class LstmGateExtendModel(models.BaseModel):
def rnn_gate(self, model_input, lstm_size, num_frames, l2_penalty=1e-8, sub_scope="", **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
## Batch normalize the input
max_frames = model_input.get_shape().as_list()[1]
emb_dim = model_input.get_shape().as_list()[2]
hidden_dim = lstm_size
with tf.variable_scope(sub_scope+'lstm_forward'):
g_recurrent_unit_forward = LstmGateModel().create_recurrent_unit(emb_dim,hidden_dim,l2_penalty)
h0 = tf.zeros([tf.shape(model_input)[0], hidden_dim])
h0 = tf.stack([h0, h0])
outputs_state = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
def _pretrain_forward(i,h_tm1,s_predictions):
x_t = model_input[:,i,:]
gate, h_t = g_recurrent_unit_forward(x_t, h_tm1)
hidden_state, c_prev = tf.unstack(h_t)
s_predictions = s_predictions.write(i,hidden_state)
return i + 1, h_t, s_predictions
_, _, state_outputs = control_flow_ops.while_loop(
cond=lambda i, _1, _2: i < max_frames,
body=_pretrain_forward,
loop_vars=(tf.constant(0, dtype=tf.int32), h0, outputs_state),
swap_memory=True)
state_outputs = state_outputs.stack()
state_outputs = tf.transpose(state_outputs, [1, 0, 2])
return state_outputs
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = FLAGS.lstm_cells
num_extend = FLAGS.moe_num_extend
shape = model_input.get_shape().as_list()
frames_sum = tf.reduce_sum(tf.abs(model_input),axis=2)
frames_true = tf.ones(tf.shape(frames_sum))
frames_false = tf.zeros(tf.shape(frames_sum))
frames_bool = tf.where(tf.greater(frames_sum, frames_false), frames_true, frames_false)
frames_bool = tf.reshape(frames_bool,[-1,shape[1],1])
with tf.variable_scope("RNN"):
outputs = self.rnn_gate(model_input, lstm_size, num_frames)
W1 = tf.Variable(tf.truncated_normal([lstm_size,num_extend], stddev=0.1), name="W1")
b1 = tf.Variable(tf.constant(0.1, shape=[num_extend]), name="b1")
W2 = tf.Variable(tf.truncated_normal([shape[2],num_extend], stddev=0.1), name="W2")
b2 = tf.Variable(tf.constant(0.1, shape=[num_extend]), name="b2")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W1))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b1))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W2))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b2))
rnn_output = tf.reshape(outputs,[-1,lstm_size])
rnn_input = tf.reshape(model_input,[-1,shape[2]])
rnn_attention = tf.nn.softmax(tf.reshape(tf.nn.xw_plus_b(rnn_output,W1,b1)+tf.nn.xw_plus_b(rnn_input,W2,b2),[-1,shape[1],num_extend]),dim=1)*frames_bool
rnn_attention = rnn_attention/(tf.reduce_sum(rnn_attention, axis=1, keep_dims=True)+1e-6)
rnn_attention = tf.transpose(rnn_attention,perm=[0,2,1])
rnn_out = tf.einsum('aij,ajk->aik', rnn_attention, outputs)
pooled = tf.reshape(rnn_out,[-1, lstm_size])
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=pooled,
vocab_size=vocab_size,
**unused_params)
class LstmGluExtendModel(models.BaseModel):
def rnn_gate(self, model_input, lstm_size, num_frames, l2_penalty=1e-8, sub_scope="", **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
## Batch normalize the input
max_frames = model_input.get_shape().as_list()[1]
emb_dim = model_input.get_shape().as_list()[2]
hidden_dim = lstm_size
with tf.variable_scope(sub_scope+'lstm_forward'):
g_recurrent_unit_forward = LstmGlu2Model().create_recurrent_unit(emb_dim, hidden_dim, l2_penalty)
h0 = tf.zeros([tf.shape(model_input)[0], hidden_dim])
h0 = tf.stack([h0, h0])
h1 = tf.zeros([tf.shape(model_input)[0], emb_dim])
h1 = tf.stack([h1, h1])
outputs_state = tensor_array_ops.TensorArray(
dtype=tf.float32, size=max_frames,
dynamic_size=False, infer_shape=True)
def _pretrain_forward(i, h_tm0, h_tm1,s_predictions):
x_t = model_input[:,i,:]
gate, h_t0, h_t1 = g_recurrent_unit_forward(x_t, h_tm0, h_tm1)
hidden_state, c_prev = tf.unstack(h_t1)
s_predictions = s_predictions.write(i,hidden_state)
return i + 1, h_t0, h_t1, s_predictions
_, _, _, state_outputs = control_flow_ops.while_loop(
cond=lambda i, _1, _2, _3: i < max_frames,
body=_pretrain_forward,
loop_vars=(tf.constant(0, dtype=tf.int32), h1, h0, outputs_state),
swap_memory=True)
state_outputs = state_outputs.stack()
state_outputs = tf.transpose(state_outputs, [1, 0, 2])
return state_outputs
def create_model(self, model_input, vocab_size, num_frames,l2_penalty=1e-8, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = FLAGS.lstm_cells
num_extend = FLAGS.moe_num_extend
shape = model_input.get_shape().as_list()
frames_sum = tf.reduce_sum(tf.abs(model_input),axis=2)
frames_true = tf.ones(tf.shape(frames_sum))
frames_false = tf.zeros(tf.shape(frames_sum))
frames_bool = tf.where(tf.greater(frames_sum, frames_false), frames_true, frames_false)
frames_bool = tf.reshape(frames_bool,[-1,shape[1],1])
with tf.variable_scope("RNN"):
outputs = self.rnn_gate(model_input, lstm_size, num_frames)
W1 = tf.Variable(tf.truncated_normal([lstm_size,num_extend], stddev=0.1), name="W1")
b1 = tf.Variable(tf.constant(0.1, shape=[num_extend]), name="b1")
W2 = tf.Variable(tf.truncated_normal([shape[2],num_extend], stddev=0.1), name="W2")
b2 = tf.Variable(tf.constant(0.1, shape=[num_extend]), name="b2")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W1))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b1))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W2))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b2))
rnn_output = tf.reshape(outputs,[-1,lstm_size])
rnn_input = tf.reshape(model_input,[-1,shape[2]])
rnn_attention = tf.nn.softmax(tf.reshape(tf.nn.xw_plus_b(rnn_output,W1,b1)+tf.nn.xw_plus_b(rnn_input,W2,b2),[-1,shape[1],num_extend]),dim=1)*frames_bool
rnn_attention = rnn_attention/(tf.reduce_sum(rnn_attention, axis=1, keep_dims=True)+1e-6)
rnn_attention = tf.transpose(rnn_attention,perm=[0, 2, 1])
rnn_out = tf.einsum('aij,ajk->aik', rnn_attention, outputs)
pooled = tf.reshape(rnn_out,[-1,lstm_size])
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=pooled,
vocab_size=vocab_size,
**unused_params)
class LstmExtendParallelModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames,l2_penalty=1e-8, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size_sys = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
num_extend = FLAGS.moe_num_extend
shape = model_input.get_shape().as_list()
frames_sum = tf.reduce_sum(tf.abs(model_input),axis=2)
frames_true = tf.ones(tf.shape(frames_sum))
frames_false = tf.zeros(tf.shape(frames_sum))
frames_bool = tf.where(tf.greater(frames_sum, frames_false), frames_true, frames_false)
frames_bool = tf.reshape(frames_bool,[-1,shape[1],1])
feature_sizes = FLAGS.feature_sizes
feature_sizes = [int(feature_size) for feature_size in feature_sizes.split(',')]
rgb_input = model_input[:,:,0:feature_sizes[0]]
audio_input = model_input[:,:,feature_sizes[0]:]
audio_input = audio_input/(tf.reduce_max(audio_input,axis=2,keep_dims=True)+1e-6)
shape_i = rgb_input.get_shape().as_list()
lstm_size = lstm_size_sys
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=False)
for _ in range(number_of_layers)
],
state_is_tuple=False)
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, rgb_input,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32)
W1 = tf.Variable(tf.truncated_normal([lstm_size,num_extend], stddev=0.1), name="W1")
b1 = tf.Variable(tf.constant(0.0, shape=[num_extend]), name="b1")
W2 = tf.Variable(tf.truncated_normal([shape_i[2],num_extend], stddev=0.1), name="W2")
b2 = tf.Variable(tf.constant(0.0, shape=[num_extend]), name="b2")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W1))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b1))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W2))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b2))
rnn_output = tf.reshape(outputs,[-1,lstm_size])
rnn_input = tf.reshape(rgb_input,[-1,shape_i[2]])
rnn_attention = tf.nn.softmax(tf.reshape(tf.nn.xw_plus_b(rnn_output,W1,b1)+tf.nn.xw_plus_b(rnn_input,W2,b2),[-1,shape_i[1],num_extend]),dim=1)*frames_bool
rnn_attention = rnn_attention/tf.reduce_sum(rnn_attention, axis=1, keep_dims=True)
rnn_attention = tf.transpose(rnn_attention,perm=[0,2,1])
rgb_out = tf.einsum('aij,ajk->aik', rnn_attention, outputs)
audio_out = tf.einsum('aij,ajk->aik', rnn_attention, audio_input)
pooled = tf.reshape(tf.concat((rgb_out,audio_out),axis=2),[-1,shape[2]])
pooled = tf.nn.l2_normalize(pooled, dim=1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=pooled,
vocab_size=vocab_size,
**unused_params)
class LstmExtendCombineModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames,l2_penalty=1e-8, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
lstm_size = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
num_extend = FLAGS.moe_num_extend
shape = model_input.get_shape().as_list()
frames_sum = tf.reduce_sum(tf.abs(model_input),axis=2)
frames_true = tf.ones(tf.shape(frames_sum))
frames_false = tf.zeros(tf.shape(frames_sum))
frames_bool = tf.where(tf.greater(frames_sum, frames_false), frames_true, frames_false)
frames_bool = tf.reshape(frames_bool,[-1,shape[1],1])
lstm_input = model_input
lstm_shape = lstm_input.get_shape().as_list()
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=False)
for _ in range(number_of_layers)
],
state_is_tuple=False)
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, lstm_input,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32)
W1 = tf.Variable(tf.truncated_normal([lstm_size,num_extend], stddev=0.1), name="W1")
b1 = tf.Variable(tf.constant(0.1, shape=[num_extend]), name="b1")
W2 = tf.Variable(tf.truncated_normal([lstm_shape[2],num_extend], stddev=0.1), name="W2")
b2 = tf.Variable(tf.constant(0.1, shape=[num_extend]), name="b2")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W1))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b1))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W2))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b2))
rnn_output = tf.reshape(outputs,[-1,lstm_size])
rnn_input = tf.reshape(lstm_input,[-1,lstm_shape[2]])
rnn_attention = tf.nn.softmax(tf.reshape(tf.nn.xw_plus_b(rnn_output,W1,b1)+tf.nn.xw_plus_b(rnn_input,W2,b2),[-1,lstm_shape[1],num_extend]),dim=1)*frames_bool
rnn_attention = rnn_attention/tf.reduce_sum(rnn_attention, axis=1, keep_dims=True)
rnn_attention = tf.transpose(rnn_attention,perm=[0,2,1])
rnn_out = tf.einsum('aij,ajk->aik', rnn_attention, outputs)
rnn_pooled = tf.reshape(rnn_out,[-1,lstm_size])
aggregated_model = getattr(video_level_models, "MoeMix4Model")
result = aggregated_model().create_model(
model_input=rnn_pooled,
vocab_size=vocab_size,
**unused_params)
final_probabilities = result["predictions"]
probabilities_by_class = result["predictions_class"]
final_probabilities = tf.reshape(final_probabilities,[-1,num_extend,vocab_size])
probabilities_by_class = tf.reshape(probabilities_by_class,[-1,num_extend,vocab_size*FLAGS.moe_layers])
result["predictions"] = tf.reduce_max(final_probabilities,axis=1)
result["predictions_class"] = tf.reduce_max(probabilities_by_class,axis=1)
return result
class LstmMoeModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params):
"""Creates a model which uses a stack of LSTMs to represent the video.
Args:
model_input: A 'batch_size' x 'max_frames' x 'num_features' matrix of
input features.
vocab_size: The number of classes in the dataset.
num_frames: A vector of length 'batch' which indicates the number of
frames for each video (before padding).
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
'batch_size' x 'num_classes'.
"""
shape = model_input.get_shape().as_list()
lstm_size = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
num_mixtures = 2
## Batch normalize the input
lstm_cell = tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=False)
with tf.variable_scope("RNN"):
states = []
layer_input = model_input
for i in range(number_of_layers):
outputs, state = tf.nn.dynamic_rnn(lstm_cell, layer_input,
sequence_length=num_frames,
swap_memory=True,
dtype=tf.float32,scope="lstm-%s" % i)
states.append(state)
if iaik', rnn_attention, outputs_features)
pooled = tf.reshape(rnn_out,[-1,lstm_size])
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
return aggregated_model().create_model(
model_input=pooled,
vocab_size=vocab_size,
**unused_params)
class VideoFrameEvalModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
num_extend = FLAGS.moe_num_extend
shape = model_input.get_shape().as_list()
frames_sum = tf.reduce_sum(tf.abs(model_input),axis=2)
random_inputs = []
random_num_frames = []
for i in range(num_extend):
frames_index = tf.reshape(tf.range(shape[1],dtype=tf.float32),[1,shape[1]])
frames_index = tf.ones(tf.shape(frames_sum))*frames_index
frames_index = tf.transpose(tf.random_shuffle(tf.transpose(frames_index)))
frames_index = frames_index[:,0:shape[1]//2]
frames_index = tf.negative(frames_index)
frames_index, indexes = tf.nn.top_k(frames_index, sorted=True, k=shape[1]//2)
frames_index = tf.negative(frames_index)
frames_valid = tf.cast(tf.reshape(num_frames,[-1,1]),dtype=tf.float32)
frames_true = tf.ones(tf.shape(frames_index))
frames_false = tf.zeros(tf.shape(frames_index))
frames_bool = tf.where(tf.less(frames_index, frames_valid), frames_true, frames_false)
random_num_frame = tf.reduce_sum(frames_bool,axis=1)
random_num_frames.append(random_num_frame)
batch_size = tf.shape(model_input)[0]
batch_index = tf.tile(
tf.expand_dims(tf.range(batch_size), 1), [1, shape[1]//2])
index = tf.stack([batch_index, tf.cast(frames_index,dtype=tf.int32)], 2)
random_input = tf.gather_nd(model_input, index)
random_inputs.append(random_input)
model_input = tf.stack(random_inputs,axis=1)
random_num_frames = tf.stack(random_num_frames,axis=1)
model_input = tf.reshape(model_input,[-1,shape[1]//2,shape[2]])
random_num_frames = tf.reshape(random_num_frames,[-1])
frames_sum = tf.reduce_sum(tf.abs(model_input),axis=2)
frames_true = tf.ones(tf.shape(frames_sum))
frames_false = tf.zeros(tf.shape(frames_sum))
frames_bool = tf.reshape(tf.where(tf.greater(frames_sum, frames_false), frames_true, frames_false),[-1,shape[1]//2,1])
model_input = tf.reduce_sum(model_input*frames_bool,axis=1)/(tf.reduce_sum(frames_bool,axis=1)+1e-6)
model_input = tf.nn.l2_normalize(model_input,dim=1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
result_frames = aggregated_model().create_model(
model_input=model_input,
vocab_size=vocab_size,
**unused_params)
predictions_frames = result_frames["predictions"]
predictions_frames = tf.reshape(predictions_frames,[-1,num_extend,vocab_size])
final_probabilities = tf.reduce_mean(predictions_frames,axis=1)
return {"predictions": final_probabilities}
class FrameAttentionEvalModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
num_extend = FLAGS.moe_num_extend
shape = model_input.get_shape().as_list()
frames_sum = tf.reduce_sum(tf.abs(model_input),axis=2)
random_inputs = []
random_num_frames = []
for i in range(num_extend):
frames_index = tf.reshape(tf.range(shape[1],dtype=tf.float32),[1,shape[1]])
frames_index = tf.ones(tf.shape(frames_sum))*frames_index
frames_index = tf.transpose(tf.random_shuffle(tf.transpose(frames_index)))
frames_index = frames_index[:,0:shape[1]//2]
frames_index = tf.negative(frames_index)
frames_index, indexes = tf.nn.top_k(frames_index, sorted=True, k=shape[1]//2)
frames_index = tf.negative(frames_index)
frames_valid = tf.cast(tf.reshape(num_frames,[-1,1]),dtype=tf.float32)
frames_true = tf.ones(tf.shape(frames_index))
frames_false = tf.zeros(tf.shape(frames_index))
frames_bool = tf.where(tf.less(frames_index, frames_valid), frames_true, frames_false)
random_num_frame = tf.reduce_sum(frames_bool,axis=1)
random_num_frames.append(random_num_frame)
batch_size = tf.shape(model_input)[0]
batch_index = tf.tile(
tf.expand_dims(tf.range(batch_size), 1), [1, shape[1]//2])
index = tf.stack([batch_index, tf.cast(frames_index,dtype=tf.int32)], 2)
random_input = tf.gather_nd(model_input, index)
random_inputs.append(random_input)
model_input = tf.stack(random_inputs,axis=1)
random_num_frames = tf.stack(random_num_frames,axis=1)
model_input = tf.reshape(model_input,[-1,shape[1]//2,shape[2]])
random_num_frames = tf.reshape(random_num_frames,[-1])
result_frames = LstmExtendModel().create_model(
model_input=model_input,
vocab_size=vocab_size,
num_frames=random_num_frames,
**unused_params)
predictions_frames = result_frames["predictions"]
predictions_frames = tf.reshape(predictions_frames,[-1,num_extend,vocab_size])
predictions_frames = tf.clip_by_value(predictions_frames,0.0,1.0)
final_probabilities = tf.reduce_mean(predictions_frames,axis=1)
return {"predictions": final_probabilities}
class FrameRandomEvalModel(models.BaseModel):
def create_model(self, model_input, vocab_size, num_frames, **unused_params):
lstm_size = FLAGS.lstm_cells
number_of_layers = FLAGS.lstm_layers
num_extend = FLAGS.moe_num_extend
shape = model_input.get_shape().as_list()
frames_sum = tf.reduce_sum(tf.abs(model_input),axis=2)
random_inputs = []
random_num_frames = []
for i in range(num_extend):
frames_index = tf.reshape(tf.range(shape[1],dtype=tf.float32),[1,shape[1]])
frames_index = tf.ones(tf.shape(frames_sum))*frames_index
frames_index = tf.transpose(tf.random_shuffle(tf.transpose(frames_index)))
frames_index = frames_index[:,0:shape[1]//2]
frames_index = tf.negative(frames_index)
frames_index, indexes = tf.nn.top_k(frames_index, sorted=True, k=shape[1]//2)
frames_index = tf.negative(frames_index)
frames_valid = tf.cast(tf.reshape(num_frames,[-1,1]),dtype=tf.float32)
frames_true = tf.ones(tf.shape(frames_index))
frames_false = tf.zeros(tf.shape(frames_index))
frames_bool = tf.where(tf.less(frames_index, frames_valid), frames_true, frames_false)
random_num_frame = tf.reduce_sum(frames_bool,axis=1)
random_num_frames.append(random_num_frame)
batch_size = tf.shape(model_input)[0]
batch_index = tf.tile(
tf.expand_dims(tf.range(batch_size), 1), [1, shape[1]//2])
index = tf.stack([batch_index, tf.cast(frames_index,dtype=tf.int32)], 2)
random_input = tf.gather_nd(model_input, index)
random_inputs.append(random_input)
model_input = tf.stack(random_inputs,axis=1)
random_num_frames = tf.stack(random_num_frames,axis=1)
model_input = tf.reshape(model_input,[-1,shape[1]//2,shape[2]])
random_num_frames = tf.reshape(random_num_frames,[-1])
## Batch normalize the input
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[
tf.contrib.rnn.BasicLSTMCell(
lstm_size, forget_bias=1.0, state_is_tuple=True)
for _ in range(number_of_layers)
],
state_is_tuple=True)
with tf.variable_scope("RNN"):
outputs, state = tf.nn.dynamic_rnn(stacked_lstm, model_input,
sequence_length=random_num_frames,
swap_memory=True,
dtype=tf.float32)
state_c = tf.concat(map(lambda x: x.c, state), axis=1)
aggregated_model = getattr(video_level_models,
FLAGS.video_level_classifier_model)
result_frames = aggregated_model().create_model(
model_input=state_c,
vocab_size=vocab_size,
**unused_params)
#final_probabilities = result_frames["predictions"]
predictions_frames = result_frames["predictions"]
predictions_frames = tf.reshape(predictions_frames,[-1,num_extend,vocab_size])
final_probabilities = tf.reduce_mean(predictions_frames,axis=1)
return {"predictions": final_probabilities}
class CnnDCCDistillChainModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def cnn(self,
model_input,
l2_penalty=1e-8,
num_filters = [1024, 1024, 1024],
filter_sizes = [1,2,3],
sub_scope="",
**unused_params):
max_frames = model_input.get_shape().as_list()[1]
num_features = model_input.get_shape().as_list()[2]
shift_inputs = []
for i in range(max(filter_sizes)):
if i == 0:
shift_inputs.append(model_input)
else:
shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:])
cnn_outputs = []
for nf, fs in zip(num_filters, filter_sizes):
sub_input = tf.concat(shift_inputs[:fs], axis=2)
sub_filter = tf.get_variable(sub_scope+"cnn-filter-len%d"%fs,
shape=[num_features*fs, nf], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(l2_penalty))
cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter))
cnn_output = tf.concat(cnn_outputs, axis=2)
return cnn_output
def create_model(self, model_input, vocab_size, num_frames, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", distill_labels=None, original_input=None, **unused_params):
num_layers = FLAGS.moe_layers
relu_cells = 256
max_frames = model_input.get_shape().as_list()[1]
relu_layers = []
support_predictions = []
mask = self.get_mask(max_frames, num_frames)
mean_input = tf.einsum("ijk,ij->ik", model_input, mask) \
/ tf.expand_dims(tf.cast(num_frames, dtype=tf.float32), dim=1)
mean_relu = slim.fully_connected(
mean_input,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"mean-relu")
mean_relu_norm = tf.nn.l2_normalize(mean_relu, dim=1)
relu_layers.append(mean_relu_norm)
cnn_output = self.cnn(model_input, num_filters=[relu_cells,relu_cells,relu_cells*2], filter_sizes=[1,2,3], sub_scope=sub_scope+"cnn0")
max_cnn_output = tf.reduce_max(cnn_output, axis=1)
normalized_cnn_output = tf.nn.l2_normalize(max_cnn_output, dim=1)
next_input = normalized_cnn_output
for layer in range(num_layers):
if layer==0:
sub_prediction = self.sub_model(next_input, vocab_size, distill_labels=distill_labels, sub_scope=sub_scope+"prediction-%d"%layer)
else:
sub_prediction = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"prediction-%d"%layer)
support_predictions.append(sub_prediction)
sub_relu = slim.fully_connected(
sub_prediction,
relu_cells,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope=sub_scope+"relu-%d"%layer)
relu_norm = tf.nn.l2_normalize(sub_relu, dim=1)
relu_layers.append(relu_norm)
cnn_output = self.cnn(model_input, num_filters=[relu_cells,relu_cells,relu_cells*2], filter_sizes=[1,2,3], sub_scope=sub_scope+"cnn%d"%(layer+1))
max_cnn_output = tf.reduce_max(cnn_output, axis=1)
normalized_cnn_output = tf.nn.l2_normalize(max_cnn_output, dim=1)
next_input = tf.concat([normalized_cnn_output] + relu_layers, axis=1)
main_predictions = self.sub_model(next_input, vocab_size, sub_scope=sub_scope+"-main")
support_predictions = tf.concat(support_predictions, axis=1)
return {"predictions": main_predictions, "predictions_class": support_predictions}
def sub_model(self, model_input, vocab_size, num_mixtures=None,
l2_penalty=1e-8, sub_scope="", distill_labels=None,**unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
class_size = 256
if distill_labels is not None:
class_input = slim.fully_connected(
distill_labels,
class_size,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="class_inputs")
class_input = tf.nn.l2_normalize(class_input, dim=1)
model_input = tf.concat((model_input, class_input),axis=1)
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-"+sub_scope)
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-"+sub_scope)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
final_probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
[-1, vocab_size])
return final_probabilities
def get_mask(self, max_frames, num_frames):
mask_array = []
for i in range(max_frames + 1):
tmp = [0.0] * max_frames
for j in range(i):
tmp[j] = 1.0
mask_array.append(tmp)
mask_array = np.array(mask_array)
mask_init = tf.constant_initializer(mask_array)
mask_emb = tf.get_variable("mask_emb", shape = [max_frames + 1, max_frames],
dtype = tf.float32, trainable = False, initializer = mask_init)
mask = tf.nn.embedding_lookup(mask_emb, num_frames)
return mask
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-distillchain_cnndcc_layer2moe4.sh
================================================
for part in ensemble_validate ensemble_train test; do
#for part in ensemble_validate; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble-distill.py \
--output_dir="/Youtube-8M/model_predictions/${part}/distillchain_cnndcc_layer2moe4" \
--model_checkpoint_path="../model/frame_level_cnndcc_distillchain_model/model.ckpt-188273" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--model=CnnDCCDistillChainModel \
--moe_layers=2 \
--moe_method=None \
--lstm_cells=1024 \
--moe_num_mixtures=4 \
--train=False \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-distillchain_cnndcc_layer2moe4_ensemble.sh
================================================
for part in ensemble_train ensemble_validate test; do
#for part in ensemble_validate; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble-distill.py \
--output_dir="/Youtube-8M/model_predictions/${part}/distillchain_cnndcc_layer2moe4" \
--model_checkpoint_path="../model/frame_level_cnndcc_distillchain_model/model.ckpt-188273" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--model=CnnDCCDistillChainModel \
--moe_layers=2 \
--moe_method=None \
--lstm_cells=1024 \
--moe_num_mixtures=4 \
--train=False \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-distillchain_lstm_extend_moe8.sh
================================================
for part in ensemble_train ensemble_validate test; do
#for part in ensemble_validate; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble-distill.py \
--output_dir="/Youtube-8M/model_predictions/${part}/distillchain_lstm_attention8max" \
--model_checkpoint_path="../model/frame_level_lstm_extend_distillchain_model/model.ckpt-74796" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--model=LstmExtendModel \
--video_level_classifier_model=MoeExtendDistillChainModel \
--moe_num_extend=8 \
--moe_method=None \
--lstm_cells=1024 \
--moe_num_mixtures=8 \
--train=False \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-distillchain_lstm_gate_moe8.sh
================================================
for part in ensemble_train ensemble_validate test; do
#for part in ensemble_validate; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble-distill.py \
--output_dir="/Youtube-8M/model_predictions/${part}/distillchain_lstm_gate_moe8" \
--model_checkpoint_path="../model/frame_level_lstm_gate_distillchain_model/model.ckpt-63344" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--model=LstmGateModel \
--video_level_classifier_model=MoeDistillChainModel \
--moe_num_extend=4 \
--moe_method=None \
--lstm_cells=1024 \
--moe_num_mixtures=8 \
--train=False \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-distillchain_lstm_gate_moe8_v2.sh
================================================
for part in ensemble_train ensemble_validate test; do
#for part in ensemble_validate; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble-distill.py \
--output_dir="/Youtube-8M/model_predictions/${part}/distillchain_v2_lstm_gate_moe8" \
--model_checkpoint_path="../model/frame_level_lstm_gate_distillchain_v2_model/model.ckpt-73400" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_v2_matrix_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--model=LstmGateModel \
--video_level_classifier_model=MoeDistillChainModel \
--moe_num_extend=4 \
--moe_method=None \
--lstm_cells=1024 \
--moe_num_mixtures=8 \
--train=False \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-distillchain_lstm_glu2_moe8_v2.sh
================================================
for part in ensemble_validate test; do
#for part in ensemble_validate; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble-distill.py \
--output_dir="/Youtube-8M/model_predictions/${part}/distillchain_v2_lstm_glu2_moe8" \
--model_checkpoint_path="../model/frame_level_lstm_glu2_distillchain_v2_model/model.ckpt-72525" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_v2_matrix_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--model=LstmGlu2Model \
--video_level_classifier_model=MoeDistillChainModel \
--moe_num_extend=4 \
--moe_method=None \
--lstm_cells=1024 \
--moe_num_mixtures=8 \
--train=False \
--batch_size=128 \
--file_size=4096
done
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-distillchain_lstm_moe8.sh
================================================
for part in ensemble_train ensemble_validate test; do
#for part in ensemble_validate; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble-distill.py \
--output_dir="/Youtube-8M/model_predictions/${part}/distillchain_lstm_moe8" \
--model_checkpoint_path="../model/frame_level_lstm_distillchain_model/model.ckpt-49873" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--model=LstmModel \
--video_level_classifier_model=MoeDistillChainModel \
--moe_num_extend=4 \
--moe_method=None \
--lstm_cells=1024 \
--lstm_layers=1 \
--moe_num_mixtures=8 \
--train=False \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-distillchain_lstm_moe8_v2.sh
================================================
for part in ensemble_train ensemble_validate test; do
#for part in ensemble_validate; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble-distill.py \
--output_dir="/Youtube-8M/model_predictions/${part}/distillchain_lstm_moe8_v2" \
--model_checkpoint_path="../model/frame_level_lstm_distillchain_v2_model/model.ckpt-49873" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation_v2/ensemble_mean_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--model=LstmModel \
--video_level_classifier_model=MoeDistillChainModel \
--moe_num_extend=4 \
--moe_method=None \
--lstm_cells=1024 \
--lstm_layers=1 \
--moe_num_mixtures=8 \
--train=False \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-distillchain_lstm_multiscale2layer_moe8.sh
================================================
for part in ensemble_train ensemble_validate test; do
#for part in ensemble_validate; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble-distill.py \
--output_dir="/Youtube-8M/model_predictions/${part}/distillchain_lstm_multiscale_layer2_moe8" \
--model_checkpoint_path="../model/frame_level_lstm_multiscale_distillchain_model/model.ckpt-103725" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--model=LstmMultiscaleDitillChainModel \
--moe_num_extend=2 \
--moe_method=None \
--lstm_cells=1024 \
--moe_num_mixtures=4 \
--cnn_cells=256 \
--train=False \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-distillchain_lstm_multiscale4layer_moe8.sh
================================================
for part in ensemble_train ensemble_validate test; do
#for part in ensemble_validate; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble-distill.py \
--output_dir="/Youtube-8M/model_predictions/${part}/distillchain_lstm_multiscale_layer4_moe8" \
--model_checkpoint_path="../model/frame_level_lstm_multiscale_distillchain4_model/model.ckpt-79645" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--model=LstmMultiscaleDitillChainModel \
--moe_num_extend=4 \
--moe_method=None \
--lstm_cells=1024 \
--moe_num_mixtures=4 \
--cnn_cells=196 \
--train=False \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-distillchain_video_norm_moe8.sh
================================================
for part in ensemble_train ensemble_validate test; do
#for part in ensemble_validate; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble-distill.py \
--output_dir="/Youtube-8M/model_predictions/${part}/distillchain_video_norm_moe8" \
--model_checkpoint_path="../model/video_level_distillchainnorm2_model/model.ckpt-13412" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--model=MoeDistillChainNorm2Model \
--moe_num_extend=4 \
--moe_method=None \
--lstm_cells=1024 \
--lstm_layers=1 \
--moe_num_mixtures=8 \
--train=False \
--batch_size=128 \
--file_size=4096
done
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-distillchain_video_norm_moe8_local.sh
================================================
#for part in ensemble_train ensemble_validate test; do
for part in ensemble_train; do
CUDA_VISIBLE_DEVICES=0 python inference-pre-ensemble-distill.py \
--output_dir="/Youtube-8M/model_predictions/${part}/distillchain_video_norm_moe8" \
--model_checkpoint_path="../model/video_level_distillchainnorm2_model/model.ckpt-13412" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--model=MoeDistillChainNorm2Model \
--moe_num_extend=4 \
--moe_method=None \
--lstm_cells=1024 \
--lstm_layers=1 \
--moe_num_mixtures=8 \
--train=False \
--batch_size=128 \
--file_size=4096
done
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-distillsplit_lstm_gate_moe8.sh
================================================
for part in ensemble_train ensemble_validate test; do
#for part in ensemble_validate; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble-distill.py \
--output_dir="/Youtube-8M/model_predictions/${part}/distillsplit_lstm_gate_moe8" \
--model_checkpoint_path="../model/frame_level_lstm_gate_distillsplit_model/model.ckpt-130570" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--distill_data_pattern="/Youtube-8M/model_predictions/${part}/distillation/ensemble_mean_model/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--distill_names="predictions" \
--feature_sizes="1024,128" \
--distill_sizes="4716" \
--model=LstmGateModel \
--video_level_classifier_model=MoeDistillSplitModel \
--moe_num_extend=4 \
--moe_method=None \
--lstm_cells=1024 \
--moe_num_mixtures=8 \
--train=False \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-lstm2_attention8_max.sh
================================================
MODEL=InputExtendModel
for part in test; do
CUDA_VISIBLE_DEVICES=1 python inference_with_rebuild.py \
--output_dir="/Youtube-8M/model_predictions/${part}/lstm2_attention8_max" \
--model_checkpoint_path="../model/frame_level_input_extend_model/model.ckpt-197005" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=32 \
--file_size=4096 \
--model=$MODEL \
--video_level_classifier_model=MoeExtendModel \
--moe_num_extend=4 \
--moe_method=None \
--lstm_cells=1024 \
--class_size=100 \
--encoder_size=100 \
--moe_num_mixtures=8 \
--moe_layers=3 \
--run_once=True
done
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-lstm_attention8_max.sh
================================================
#for part in ensemble_train ensemble_validate test; do
for part in ensemble_validate; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/lstm_attention8_max" \
--model_checkpoint_path="../model/frame_level_lstm_extend8_model/model.ckpt-181785" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-lstm_attention_max_mean.sh
================================================
MODEL=FrameAttentionEvalModel
for part in ensemble_validate; do
CUDA_VISIBLE_DEVICES=1 python inference_with_rebuild.py \
--output_dir="/Youtube-8M/model_predictions/${part}/lstm_attention8_max_mean" \
--model_checkpoint_path="../model/frame_level_lstm_extend8_model/model.ckpt-181785" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=32 \
--file_size=4096 \
--model=$MODEL \
--video_level_classifier_model=MoeExtendModel \
--moe_num_extend=8 \
--moe_method=None \
--lstm_cells=1024 \
--class_size=100 \
--encoder_size=100 \
--moe_num_mixtures=8 \
--moe_layers=3 \
--run_once=True
done
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-lstm_gate_multiscale4_moe4.sh
================================================
for part in ensemble_train ensemble_validate test; do
#for part in ensemble_validate test; do
CUDA_VISIBLE_DEVICES=0 python inference_with_rebuild.py \
--output_dir="/Youtube-8M/model_predictions/${part}/lstm_gate_multiscale4_moe4" \
--model_checkpoint_path="../model/frame_level_lstm_multiscale2_model/model.ckpt-193803" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=LstmMultiscale2Model \
--video_level_classifier_model=MoeModel \
--moe_num_extend=4 \
--moe_method=None \
--lstm_cells=1024 \
--moe_num_mixtures=4 \
--norm=False \
--train=False \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-lstm_multiscale4_moe4.sh
================================================
#for part in ensemble_train ensemble_validate test; do
for part in ensemble_validate test; do
CUDA_VISIBLE_DEVICES=0 python inference_with_rebuild.py \
--output_dir="/Youtube-8M/model_predictions/${part}/lstm_multiscale4_moe4" \
--model_checkpoint_path="../model/frame_level_lstm_multiscale_model/model.ckpt-227882" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=LstmMultiscaleModel \
--video_level_classifier_model=MoeModel \
--moe_num_extend=4 \
--moe_method=None \
--lstm_cells=1024 \
--moe_num_mixtures=4 \
--train=False \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-lstm_random_mean_moe8.sh
================================================
MODEL=FrameRandomEvalModel
for part in ensemble_train ensemble_validate test; do
CUDA_VISIBLE_DEVICES=1 python inference_with_rebuild.py \
--output_dir="/Youtube-8M/model_predictions/${part}/lstm_random_mean_moe8" \
--model_checkpoint_path="../model/frame_level_lstm_random_model/model.ckpt-145850" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=32 \
--file_size=4096 \
--model=$MODEL \
--video_level_classifier_model=MoeModel \
--moe_num_extend=16 \
--moe_method=None \
--lstm_cells=1024 \
--class_size=100 \
--encoder_size=100 \
--moe_num_mixtures=8 \
--moe_layers=3 \
--run_once=True
done
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-lstm_shortlayers_moe8.sh
================================================
for part in ensemble_train ensemble_validate test; do
#for part in train_samples; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/lstm_shortlayers_moe8" \
--model_checkpoint_path="../model/frame_level_lstm_layer_model/model.ckpt-107188" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-lstmbiglu_1024_moe8.sh
================================================
for part in ensemble_train ensemble_validate test; do
#for part in ensemble_validate test; do
CUDA_VISIBLE_DEVICES=1 python inference_with_rebuild.py \
--output_dir="/Youtube-8M/model_predictions/${part}/lstmbiglu_cell1024_layer1_moe8" \
--model_checkpoint_path="../model/frame_level_lstm_biglu_model/model.ckpt-130983" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=LstmBigluModel \
--video_level_classifier_model=MoeModel \
--moe_num_extend=4 \
--moe_method=None \
--lstm_cells=1024 \
--moe_num_mixtures=4 \
--train=False \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-lstmgate1024_moe8.sh
================================================
for part in ensemble_train ensemble_validate test; do
#for part in ensemble_validate test; do
CUDA_VISIBLE_DEVICES=0 python inference_with_rebuild.py \
--output_dir="/Youtube-8M/model_predictions/${part}/lstmgate_cell1024_layer1_moe8" \
--model_checkpoint_path="../model/frame_level_lstm_gate_model/model.ckpt-151107" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=LstmGateModel \
--video_level_classifier_model=MoeModel \
--moe_num_extend=4 \
--moe_method=None \
--lstm_cells=1024 \
--moe_num_mixtures=8 \
--train=False \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-lstmglu2_1024_moe8.sh
================================================
for part in ensemble_train ensemble_validate test; do
#for part in ensemble_validate test; do
CUDA_VISIBLE_DEVICES=1 python inference_with_rebuild.py \
--output_dir="/Youtube-8M/model_predictions/${part}/lstmglu2_cell1024_layer1_moe8" \
--model_checkpoint_path="../model/frame_level_lstm_glu2_model/model.ckpt-132334" \
--input_data_pattern="/Youtube-8M/data/frame/${part}/*.tfrecord" \
--frame_features=True \
--feature_names="rgb,audio" \
--feature_sizes="1024,128" \
--model=LstmGlu2Model \
--video_level_classifier_model=MoeModel \
--moe_num_extend=4 \
--moe_method=None \
--lstm_cells=1024 \
--moe_num_mixtures=8 \
--train=False \
--batch_size=32 \
--file_size=4096
done
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-video_notzero_combine_chain.sh
================================================
#for part in ensemble_train ensemble_validate test; do
for part in test; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/video_notzero_combine_chain" \
--model_checkpoint_path="../model/video_level_moemix4_model/model.ckpt-14668" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--file_size=4096
done
#for part in ensemble_train ensemble_validate test; do
for part in test; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/video_weight_combine_chain" \
--model_checkpoint_path="../model/video_level_moemix4_weight_model/model.ckpt-17415" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--file_size=4096
done
#for part in ensemble_train ensemble_validate test; do
for part in test; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/video_knowledge_combine_chain" \
--model_checkpoint_path="../model/video_level_moeknowledge_model/model.ckpt-9606" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--file_size=4096
done
#for part in ensemble_train ensemble_validate test; do
for part in test; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/video_softmax_combine_chain" \
--model_checkpoint_path="../model/video_level_moesoftmax_model/model.ckpt-9501" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--file_size=4096
done
================================================
FILE: youtube-8m-zhangteng/infer_scripts/infer-video_relabel_combine_chain.sh
================================================
for part in ensemble_train ensemble_validate test; do
#for part in test; do
CUDA_VISIBLE_DEVICES=1 python inference-pre-ensemble.py \
--output_dir="/Youtube-8M/model_predictions/${part}/video_relabel_combine_chain" \
--model_checkpoint_path="../model/video_level_moemix4_relabel_model/model.ckpt-14863" \
--input_data_pattern="/Youtube-8M/data/video/${part}/*.tfrecord" \
--frame_features=False \
--feature_names="mean_rgb,mean_audio" \
--feature_sizes="1024,128" \
--batch_size=128 \
--file_size=4096
done
================================================
FILE: youtube-8m-zhangteng/inference-pre-ensemble-distill.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for generating predictions over a set of videos."""
import os
import time
import numpy
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import eval_util
import losses
import frame_level_models
import video_level_models
import readers
import utils
import numpy as np
FLAGS = flags.FLAGS
if __name__ == '__main__':
flags.DEFINE_string("model_checkpoint_path", "",
"The file path to load the model from.")
flags.DEFINE_string("output_dir", "",
"The file to save the predictions to.")
flags.DEFINE_string(
"input_data_pattern", "",
"File glob defining the evaluation dataset in tensorflow.SequenceExample "
"format. The SequenceExamples are expected to have an 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
flags.DEFINE_string(
"distill_data_pattern", "",
"File glob defining the distill dataset in tensorflow.SequenceExample "
"format. The SequenceExamples are expected to have an 'predictions' byte array "
"sequence feature.")
flags.DEFINE_string("distill_names", "predictions", "Name of the distill feature "
"to use for inference.")
flags.DEFINE_string("distill_sizes", "4716", "Length of the distill feature vectors.")
# Model flags.
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Which loss function to use for training the model.")
flags.DEFINE_bool(
"frame_features", False,
"If set, then --eval_data_pattern must be frame-level features. "
"Otherwise, --eval_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_bool(
"norm", True,
"If set, then --input_data should be l2-normalized before follow-up processing. "
"Otherwise, --input_data remain unchanged")
flags.DEFINE_string(
"model", "LogisticModel",
"Which architecture to use for the model. Options include 'Logistic', "
"'SingleMixtureMoe', and 'TwoLayerSigmoid'. See aggregated_models.py and "
"frame_level_models.py for the model definitions.")
flags.DEFINE_integer(
"batch_size", 8192,
"How many examples to process per batch.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024", "Length of the feature vectors.")
flags.DEFINE_integer("file_size", 4096,
"Number of samples to be written into one tfrecord file.")
# Other flags.
flags.DEFINE_integer("num_readers", 1,
"How many threads to use for reading input files.")
flags.DEFINE_integer("top_k", 20,
"How many predictions to output per video.")
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def get_input_data_tensors(reader, data_pattern, batch_size, num_readers=1):
"""Creates the section of the graph which reads the input data.
Args:
reader: A class which parses the input data.
data_pattern: A 'glob' style path to the data files.
batch_size: How many examples to process at a time.
num_readers: How many I/O threads to use.
Returns:
A tuple containing the features tensor, labels tensor, and optionally a
tensor containing the number of frames per video. The exact dimensions
depend on the reader being used.
Raises:
IOError: If no files matching the given pattern were found.
"""
with tf.name_scope("input"):
files = gfile.Glob(data_pattern)
files.sort()
if not files:
raise IOError("Unable to find input files. data_pattern='" +
data_pattern + "'")
logging.info("number of input files: " + str(len(files)))
filename_queue = tf.train.string_input_producer(
files, num_epochs=1, shuffle=False)
examples_and_labels = [reader.prepare_reader(filename_queue)
for _ in range(num_readers)]
video_id_batch, video_batch, unused_labels, num_frames_batch = (
tf.train.batch_join(examples_and_labels,
batch_size=batch_size,
allow_smaller_final_batch=True,
enqueue_many=True))
return video_id_batch, video_batch, unused_labels, num_frames_batch
def build_graph(reader1,
reader2,
model,
eval_data_pattern,
distill_data_pattern,
label_loss_fn,
batch_size=1024,
num_readers=1):
"""Creates the Tensorflow graph for evaluation.
Args:
reader: The data file reader. It should inherit from BaseReader.
model: The core model (e.g. logistic or neural net). It should inherit
from BaseModel.
eval_data_pattern: glob path to the evaluation data files.
label_loss_fn: What kind of loss to apply to the model. It should inherit
from BaseLoss.
batch_size: How many examples to process at a time.
num_readers: How many threads to use for I/O operations.
"""
global_step = tf.Variable(0, trainable=False, name="global_step")
video_id_batch, model_input_raw, labels_batch, num_frames = get_input_data_tensors( # pylint: disable=g-line-too-long
reader1,
eval_data_pattern,
batch_size=batch_size,
num_readers=num_readers)
unused_id_batch, labels_distill, unused_labels_batch, unusednum_frames = get_input_data_tensors( # pylint: disable=g-line-too-long
reader2,
distill_data_pattern,
batch_size=batch_size,
num_readers=num_readers)
tf.summary.histogram("model_input_raw", model_input_raw)
feature_dim = len(model_input_raw.get_shape()) - 1
# Normalize input features.
if FLAGS.norm:
model_input = tf.nn.l2_normalize(model_input_raw, feature_dim)
else:
model_input = model_input_raw
with tf.name_scope("model"):
result = model.create_model(model_input,
num_frames=num_frames,
vocab_size=reader1.num_classes,
labels=labels_batch,
distill_labels=labels_distill,
is_training=False)
predictions = result["predictions"]
tf.summary.histogram("model_activations", predictions)
if "loss" in result.keys():
label_loss = result["loss"]
else:
label_loss = label_loss_fn.calculate_loss(predictions, labels_batch)
tf.add_to_collection("global_step", global_step)
tf.add_to_collection("loss", label_loss)
tf.add_to_collection("predictions", predictions)
tf.add_to_collection("input_batch", model_input)
tf.add_to_collection("video_id_batch", video_id_batch)
tf.add_to_collection("num_frames", num_frames)
tf.add_to_collection("labels", tf.cast(labels_batch, tf.float32))
tf.add_to_collection("summary_op", tf.summary.merge_all())
def inference(video_id_batch, prediction_batch, label_batch, saver, out_file_location):
global_step_val = -1
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.4)
#with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
with tf.Session() as sess:
if FLAGS.model_checkpoint_path:
checkpoint = FLAGS.model_checkpoint_path
else:
checkpoint = tf.train.latest_checkpoint(FLAGS.train_dir)
if checkpoint:
logging.info("Loading checkpoint for eval: " + checkpoint)
# Restores from checkpoint
saver.restore(sess, checkpoint)
# Assuming model_checkpoint_path looks something like:
# /my-favorite-path/yt8m_train/model.ckpt-0, extract global_step from it.
global_step_val = checkpoint.split("/")[-1].split("-")[-1]
else:
logging.info("No checkpoint file found.")
return global_step_val
sess.run([tf.local_variables_initializer()])
# Workaround for num_epochs issue.
def set_up_init_ops(variables):
init_op_list = []
for variable in list(variables):
if "train_input" in variable.name:
init_op_list.append(tf.assign(variable, 1))
variables.remove(variable)
init_op_list.append(tf.variables_initializer(variables))
return init_op_list
sess.run(set_up_init_ops(tf.get_collection_ref(
tf.GraphKeys.LOCAL_VARIABLES)))
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
num_examples_processed = 0
start_time = time.time()
video_id = []
video_label = []
video_features = []
filenum = 0
directory = FLAGS.output_dir
if not os.path.exists(directory):
os.makedirs(directory)
else:
raise IOError("Output path exists! path='" + directory + "'")
try:
while not coord.should_stop():
video_id_batch_val, video_label_batch_val, predictions = sess.run([video_id_batch, label_batch, prediction_batch])
now = time.time()
num_examples_processed += len(video_id_batch_val)
video_id.append(video_id_batch_val)
video_label.append(video_label_batch_val)
video_features.append(predictions)
if num_examples_processed>=FLAGS.file_size:
assert num_examples_processed==FLAGS.file_size, "num_examples_processed should be equal to file_size"
video_id = np.concatenate(video_id,axis=0)
video_label = np.concatenate(video_label,axis=0)
video_features = np.concatenate(video_features,axis=0)
write_to_record(video_id, video_label, video_features, filenum, num_examples_processed)
filenum += 1
video_id = []
video_label = []
video_features = []
num_examples_processed = 0
logging.info("num examples processed: " + str(num_examples_processed) + " elapsed seconds: " + "{0:.2f}".format(now-start_time))
except tf.errors.OutOfRangeError:
logging.info('Done with inference. The output file was written to ' + out_file_location)
finally:
coord.request_stop()
if num_examples_processed=FLAGS.file_size:
assert num_examples_processed==FLAGS.file_size, "num_examples_processed should be equal to file_size"
video_id = np.concatenate(video_id,axis=0)
video_label = np.concatenate(video_label,axis=0)
video_inputs = np.concatenate(video_inputs,axis=0)
video_features = np.concatenate(video_features,axis=0)
write_to_record(video_id, video_label, video_inputs, video_features, filenum, num_examples_processed)
filenum += 1
video_id = []
video_label = []
video_inputs = []
video_features = []
num_examples_processed = 0
logging.info("num examples processed: " + str(num_examples_processed) + " elapsed seconds: " + "{0:.2f}".format(now-start_time))
except tf.errors.OutOfRangeError:
logging.info('Done with inference. The output file was written to ' + out_file_location)
finally:
coord.request_stop()
if num_examples_processed=FLAGS.file_size:
if num_examples_processed>FLAGS.file_size:
print("Wrong!", num_examples_processed)
else:
print(num_examples_processed)
#logging.info("num examples processed: " + str(num_examples_processed) + " elapsed seconds: " + "{0:.2f}".format(now-start_time))
"""
thefile = open('inference_test/video_id_test_'+str(filenum)+'.out', 'w')
for item in video_id:
item = ''.join(str(e) for e in item)
thefile.write("%s\n" % item)"""
filenum += 1
video_id = []
num_examples_processed = 0
except tf.errors.OutOfRangeError:
logging.info('Done with inference. The output file was written to ' + out_file_location)
finally:
coord.request_stop()
if num_examples_processed=FLAGS.file_size:
assert num_examples_processed==FLAGS.file_size, "num_examples_processed should be equal to file_size"
video_id = np.concatenate(video_id,axis=0)
video_label = np.concatenate(video_label,axis=0)
video_features = np.concatenate(video_features,axis=0)
write_to_record(video_id, video_label, video_features, filenum, num_examples_processed)
filenum += 1
video_id = []
video_label = []
video_features = []
num_examples_processed = 0
logging.info("num examples processed: " + str(num_examples_processed) + " elapsed seconds: " + "{0:.2f}".format(now-start_time))
except tf.errors.OutOfRangeError:
logging.info('Done with inference. The output file was written to ' + out_file_location)
finally:
coord.request_stop()
if num_examples_processed 0, "self.feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(self.feature_names), len(self.feature_sizes))
feature_map = {"video_id": tf.FixedLenFeature([], tf.string),
"labels": tf.VarLenFeature(tf.int64)}
for feature_index in range(num_features):
feature_map[self.feature_names[feature_index]] = tf.FixedLenFeature(
[self.feature_sizes[feature_index]], tf.float32)
features = tf.parse_example(serialized_examples, features=feature_map)
labels = tf.sparse_to_indicator(features["labels"], self.num_classes)
labels.set_shape([None, self.num_classes])
concatenated_features = tf.concat([
features[feature_name] for feature_name in self.feature_names], 1)
return features["video_id"], concatenated_features, labels, tf.ones([tf.shape(serialized_examples)[0]])
class YT8MFrameFeatureReader(BaseReader):
"""Reads TFRecords of SequenceExamples.
The TFRecords must contain SequenceExamples with the sparse in64 'labels'
context feature and a fixed length byte-quantized feature vector, obtained
from the features in 'feature_names'. The quantized features will be mapped
back into a range between min_quantized_value and max_quantized_value.
"""
def __init__(self,
num_classes=4716,
feature_sizes=[1024],
feature_names=["inc3"],
max_frames=300):
"""Construct a YT8MFrameFeatureReader.
Args:
num_classes: a positive integer for the number of classes.
feature_sizes: positive integer(s) for the feature dimensions as a list.
feature_names: the feature name(s) in the tensorflow record as a list.
max_frames: the maximum number of frames to process.
"""
assert len(feature_names) == len(feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(feature_names), len(feature_sizes))
self.num_classes = num_classes
self.feature_sizes = feature_sizes
self.feature_names = feature_names
self.max_frames = max_frames
def get_video_matrix(self,
features,
feature_size,
max_frames,
max_quantized_value,
min_quantized_value):
"""Decodes features from an input string and quantizes it.
Args:
features: raw feature values
feature_size: length of each frame feature vector
max_frames: number of frames (rows) in the output feature_matrix
max_quantized_value: the maximum of the quantized value.
min_quantized_value: the minimum of the quantized value.
Returns:
feature_matrix: matrix of all frame-features
num_frames: number of frames in the sequence
"""
decoded_features = tf.reshape(
tf.cast(tf.decode_raw(features, tf.uint8), tf.float32),
[-1, feature_size])
num_frames = tf.minimum(tf.shape(decoded_features)[0], max_frames)
feature_matrix = utils.Dequantize(decoded_features,
max_quantized_value,
min_quantized_value)
feature_matrix = resize_axis(feature_matrix, 0, max_frames)
return feature_matrix, num_frames
def prepare_reader(self,
filename_queue,
max_quantized_value=2,
min_quantized_value=-2):
"""Creates a single reader thread for YouTube8M SequenceExamples.
Args:
filename_queue: A tensorflow queue of filename locations.
max_quantized_value: the maximum of the quantized value.
min_quantized_value: the minimum of the quantized value.
Returns:
A tuple of video indexes, video features, labels, and padding data.
"""
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
contexts, features = tf.parse_single_sequence_example(
serialized_example,
context_features={"video_id": tf.FixedLenFeature(
[], tf.string),
"labels": tf.VarLenFeature(tf.int64)},
sequence_features={
feature_name : tf.FixedLenSequenceFeature([], dtype=tf.string)
for feature_name in self.feature_names
})
# read ground truth labels
labels = (tf.cast(
tf.sparse_to_dense(contexts["labels"].values, (self.num_classes,), 1,
validate_indices=False),
tf.bool))
# loads (potentially) different types of features and concatenates them
num_features = len(self.feature_names)
assert num_features > 0, "No feature selected: feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(self.feature_names), len(self.feature_sizes))
num_frames = -1 # the number of frames in the video
feature_matrices = [None] * num_features # an array of different features
for feature_index in range(num_features):
feature_matrix, num_frames_in_this_feature = self.get_video_matrix(
features[self.feature_names[feature_index]],
self.feature_sizes[feature_index],
self.max_frames,
max_quantized_value,
min_quantized_value)
if num_frames == -1:
num_frames = num_frames_in_this_feature
else:
tf.assert_equal(num_frames, num_frames_in_this_feature)
feature_matrices[feature_index] = feature_matrix
# cap the number of frames at self.max_frames
num_frames = tf.minimum(num_frames, self.max_frames)
# concatenate different features
video_matrix = tf.concat(feature_matrices, 1)
# convert to batch format.
# TODO: Do proper batch reads to remove the IO bottleneck.
batch_video_ids = tf.expand_dims(contexts["video_id"], 0)
batch_video_matrix = tf.expand_dims(video_matrix, 0)
batch_labels = tf.expand_dims(labels, 0)
batch_frames = tf.expand_dims(num_frames, 0)
return batch_video_ids, batch_video_matrix, batch_labels, batch_frames
class YT8MAggregatedDistillationFeatureReader(BaseReader):
"""Reads TFRecords of pre-aggregated Examples.
The TFRecords must contain Examples with a sparse int64 'labels' feature and
a fixed length float32 feature, obtained from the features in 'feature_name'.
The float features are assumed to be an average of dequantized values.
"""
def __init__(self,
num_classes=4716,
feature_sizes=[1024],
feature_names=["mean_inc3"]):
"""Construct a YT8MAggregatedFeatureReader.
Args:
num_classes: a positive integer for the number of classes.
feature_sizes: positive integer(s) for the feature dimensions as a list.
feature_names: the feature name(s) in the tensorflow record as a list.
"""
assert len(feature_names) == len(feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(feature_names), len(feature_sizes))
self.num_classes = num_classes
self.feature_sizes = feature_sizes
self.feature_names = feature_names
def prepare_reader(self, filename_queue, batch_size=1024):
"""Creates a single reader thread for pre-aggregated YouTube 8M Examples.
Args:
filename_queue: A tensorflow queue of filename locations.
Returns:
A tuple of video indexes, features, labels, and padding data.
"""
reader = tf.TFRecordReader()
_, serialized_examples = reader.read_up_to(filename_queue, batch_size)
# set the mapping from the fields to data types in the proto
num_features = len(self.feature_names)
assert num_features > 0, "self.feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(self.feature_names), len(self.feature_sizes))
feature_map = {"video_id": tf.FixedLenFeature([], tf.string),
"predictions": tf.FixedLenFeature([self.num_classes], tf.float32),
"labels": tf.VarLenFeature(tf.int64)}
for feature_index in range(num_features):
feature_map[self.feature_names[feature_index]] = tf.FixedLenFeature(
[self.feature_sizes[feature_index]], tf.float32)
features = tf.parse_example(serialized_examples, features=feature_map)
labels = tf.sparse_to_indicator(features["labels"], self.num_classes)
labels.set_shape([None, self.num_classes])
concatenated_features = tf.concat([
features[feature_name] for feature_name in self.feature_names], 1)
return features["video_id"], concatenated_features, labels, tf.ones([tf.shape(serialized_examples)[0]]), features["predictions"]
class YT8MDistillationFeatureReader(BaseReader):
"""Reads TFRecords of pre-aggregated Examples.
The TFRecords must contain Examples with a sparse int64 'labels' feature and
a fixed length float32 feature, obtained from the features in 'feature_name'.
The float features are assumed to be an average of dequantized values.
"""
def __init__(self,
num_classes=4716,
feature_sizes=[1024],
feature_names=["mean_inc3"]):
"""Construct a YT8MAggregatedFeatureReader.
Args:
num_classes: a positive integer for the number of classes.
feature_sizes: positive integer(s) for the feature dimensions as a list.
feature_names: the feature name(s) in the tensorflow record as a list.
"""
assert len(feature_names) == len(feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(feature_names), len(feature_sizes))
self.num_classes = num_classes
self.feature_sizes = feature_sizes
self.feature_names = feature_names
def prepare_reader(self, filename_queue, batch_size=1024):
"""Creates a single reader thread for pre-aggregated YouTube 8M Examples.
Args:
filename_queue: A tensorflow queue of filename locations.
Returns:
A tuple of video indexes, features, labels, and padding data.
"""
reader = tf.TFRecordReader()
_, serialized_examples = reader.read_up_to(filename_queue, batch_size)
# set the mapping from the fields to data types in the proto
num_features = len(self.feature_names)
assert num_features > 0, "self.feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(self.feature_names), len(self.feature_sizes))
feature_map = {"video_id": tf.FixedLenFeature([], tf.string),
"predictions": tf.FixedLenFeature([self.num_classes], tf.float32),
"labels": tf.VarLenFeature(tf.int64)}
features = tf.parse_example(serialized_examples, features=feature_map)
return features["predictions"]
class YT8MFrameDistillationFeatureReader(BaseReader):
"""Reads TFRecords of SequenceExamples.
The TFRecords must contain SequenceExamples with the sparse in64 'labels'
context feature and a fixed length byte-quantized feature vector, obtained
from the features in 'feature_names'. The quantized features will be mapped
back into a range between min_quantized_value and max_quantized_value.
"""
def __init__(self,
num_classes=4716,
feature_sizes=[1024],
feature_names=["inc3"],
max_frames=300):
"""Construct a YT8MFrameFeatureReader.
Args:
num_classes: a positive integer for the number of classes.
feature_sizes: positive integer(s) for the feature dimensions as a list.
feature_names: the feature name(s) in the tensorflow record as a list.
max_frames: the maximum number of frames to process.
"""
assert len(feature_names) == len(feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(feature_names), len(feature_sizes))
self.num_classes = num_classes
self.feature_sizes = feature_sizes
self.feature_names = feature_names
self.max_frames = max_frames
def get_video_matrix(self,
features,
feature_size,
max_frames,
max_quantized_value,
min_quantized_value):
"""Decodes features from an input string and quantizes it.
Args:
features: raw feature values
feature_size: length of each frame feature vector
max_frames: number of frames (rows) in the output feature_matrix
max_quantized_value: the maximum of the quantized value.
min_quantized_value: the minimum of the quantized value.
Returns:
feature_matrix: matrix of all frame-features
num_frames: number of frames in the sequence
"""
decoded_features = tf.reshape(
tf.cast(tf.decode_raw(features, tf.uint8), tf.float32),
[-1, feature_size])
num_frames = tf.minimum(tf.shape(decoded_features)[0], max_frames)
feature_matrix = utils.Dequantize(decoded_features,
max_quantized_value,
min_quantized_value)
feature_matrix = resize_axis(feature_matrix, 0, max_frames)
return feature_matrix, num_frames
def prepare_reader(self,
filename_queue,
max_quantized_value=2,
min_quantized_value=-2):
"""Creates a single reader thread for YouTube8M SequenceExamples.
Args:
filename_queue: A tensorflow queue of filename locations.
max_quantized_value: the maximum of the quantized value.
min_quantized_value: the minimum of the quantized value.
Returns:
A tuple of video indexes, video features, labels, and padding data.
"""
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
contexts, features = tf.parse_single_sequence_example(
serialized_example,
context_features={"video_id": tf.FixedLenFeature([], tf.string),
"predictions": tf.FixedLenFeature([self.num_classes], tf.float32),
"labels": tf.VarLenFeature(tf.int64)},
sequence_features={
feature_name : tf.FixedLenSequenceFeature([], dtype=tf.string)
for feature_name in self.feature_names
})
# read ground truth labels
labels = (tf.cast(
tf.sparse_to_dense(contexts["labels"].values, (self.num_classes,), 1,
validate_indices=False),
tf.bool))
predictions = contexts["predictions"]
# loads (potentially) different types of features and concatenates them
num_features = len(self.feature_names)
assert num_features > 0, "No feature selected: feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(self.feature_names), len(self.feature_sizes))
num_frames = -1 # the number of frames in the video
feature_matrices = [None] * num_features # an array of different features
for feature_index in range(num_features):
feature_matrix, num_frames_in_this_feature = self.get_video_matrix(
features[self.feature_names[feature_index]],
self.feature_sizes[feature_index],
self.max_frames,
max_quantized_value,
min_quantized_value)
if num_frames == -1:
num_frames = num_frames_in_this_feature
else:
tf.assert_equal(num_frames, num_frames_in_this_feature)
feature_matrices[feature_index] = feature_matrix
# cap the number of frames at self.max_frames
num_frames = tf.minimum(num_frames, self.max_frames)
# concatenate different features
video_matrix = tf.concat(feature_matrices, 1)
# convert to batch format.
# TODO: Do proper batch reads to remove the IO bottleneck.
batch_video_ids = tf.expand_dims(contexts["video_id"], 0)
batch_video_matrix = tf.expand_dims(video_matrix, 0)
batch_labels = tf.expand_dims(labels, 0)
batch_frames = tf.expand_dims(num_frames, 0)
batch_predictions = tf.expand_dims(predictions, 0)
return batch_video_ids, batch_video_matrix, batch_labels, batch_frames, batch_predictions
================================================
FILE: youtube-8m-zhangteng/rnn_residual.py
================================================
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import tensorflow as tf
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import tensor_shape
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import control_flow_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import rnn_cell_impl
from tensorflow.python.ops import tensor_array_ops
from tensorflow.python.ops import variable_scope as vs
from tensorflow.python.util import nest
# pylint: disable=protected-access
_state_size_with_prefix = rnn_cell_impl._state_size_with_prefix
# pylint: enable=protected-access
def _infer_state_dtype(explicit_dtype, state):
"""Infer the dtype of an RNN state.
Args:
explicit_dtype: explicitly declared dtype or None.
state: RNN's hidden state. Must be a Tensor or a nested iterable containing
Tensors.
Returns:
dtype: inferred dtype of hidden state.
Raises:
ValueError: if `state` has heterogeneous dtypes or is empty.
"""
if explicit_dtype is not None:
return explicit_dtype
elif nest.is_sequence(state):
inferred_dtypes = [element.dtype for element in nest.flatten(state)]
if not inferred_dtypes:
raise ValueError("Unable to infer dtype from empty state.")
all_same = all([x == inferred_dtypes[0] for x in inferred_dtypes])
if not all_same:
raise ValueError(
"State has tensors of different inferred_dtypes. Unable to infer a "
"single representative dtype.")
return inferred_dtypes[0]
else:
return state.dtype
def _on_device(fn, device):
"""Build the subgraph defined by lambda `fn` on `device` if it's not None."""
if device:
with ops.device(device):
return fn()
else:
return fn()
# pylint: disable=unused-argument
def _rnn_step(
time, sequence_length, min_sequence_length, max_sequence_length,
zero_output, state, call_cell, state_size, skip_conditionals=False):
"""Calculate one step of a dynamic RNN minibatch.
Returns an (output, state) pair conditioned on the sequence_lengths.
When skip_conditionals=False, the pseudocode is something like:
if t >= max_sequence_length:
return (zero_output, state)
if t < min_sequence_length:
return call_cell()
# Selectively output zeros or output, old state or new state depending
# on if we've finished calculating each row.
new_output, new_state = call_cell()
final_output = np.vstack([
zero_output if time >= sequence_lengths[r] else new_output_r
for r, new_output_r in enumerate(new_output)
])
final_state = np.vstack([
state[r] if time >= sequence_lengths[r] else new_state_r
for r, new_state_r in enumerate(new_state)
])
return (final_output, final_state)
Args:
time: Python int, the current time step
sequence_length: int32 `Tensor` vector of size [batch_size]
min_sequence_length: int32 `Tensor` scalar, min of sequence_length
max_sequence_length: int32 `Tensor` scalar, max of sequence_length
zero_output: `Tensor` vector of shape [output_size]
state: Either a single `Tensor` matrix of shape `[batch_size, state_size]`,
or a list/tuple of such tensors.
call_cell: lambda returning tuple of (new_output, new_state) where
new_output is a `Tensor` matrix of shape `[batch_size, output_size]`.
new_state is a `Tensor` matrix of shape `[batch_size, state_size]`.
state_size: The `cell.state_size` associated with the state.
skip_conditionals: Python bool, whether to skip using the conditional
calculations. This is useful for `dynamic_rnn`, where the input tensor
matches `max_sequence_length`, and using conditionals just slows
everything down.
Returns:
A tuple of (`final_output`, `final_state`) as given by the pseudocode above:
final_output is a `Tensor` matrix of shape [batch_size, output_size]
final_state is either a single `Tensor` matrix, or a tuple of such
matrices (matching length and shapes of input `state`).
Raises:
ValueError: If the cell returns a state tuple whose length does not match
that returned by `state_size`.
"""
# Convert state to a list for ease of use
flat_state = nest.flatten(state)
flat_zero_output = nest.flatten(zero_output)
def _copy_one_through(output, new_output):
copy_cond = (time >= sequence_length)
return _on_device(
lambda: array_ops.where(copy_cond, output, new_output),
device=new_output.op.device)
def _copy_some_through(flat_new_output, flat_new_state):
# Use broadcasting select to determine which values should get
# the previous state & zero output, and which values should get
# a calculated state & output.
flat_new_output = [
_copy_one_through(zero_output, new_output)
for zero_output, new_output in zip(flat_zero_output, flat_new_output)]
flat_new_state = [
_copy_one_through(state, new_state)
for state, new_state in zip(flat_state, flat_new_state)]
return flat_new_output + flat_new_state
def _maybe_copy_some_through():
"""Run RNN step. Pass through either no or some past state."""
new_output, new_state = call_cell()
nest.assert_same_structure(state, new_state)
flat_new_state = nest.flatten(new_state)
flat_new_output = nest.flatten(new_output)
return control_flow_ops.cond(
# if t < min_seq_len: calculate and return everything
time < min_sequence_length, lambda: flat_new_output + flat_new_state,
# else copy some of it through
lambda: _copy_some_through(flat_new_output, flat_new_state))
# TODO(ebrevdo): skipping these conditionals may cause a slowdown,
# but benefits from removing cond() and its gradient. We should
# profile with and without this switch here.
if skip_conditionals:
# Instead of using conditionals, perform the selective copy at all time
# steps. This is faster when max_seq_len is equal to the number of unrolls
# (which is typical for dynamic_rnn).
new_output, new_state = call_cell()
nest.assert_same_structure(state, new_state)
new_state = nest.flatten(new_state)
new_output = nest.flatten(new_output)
final_output_and_state = _copy_some_through(new_output, new_state)
else:
empty_update = lambda: flat_zero_output + flat_state
final_output_and_state = control_flow_ops.cond(
# if t >= max_seq_len: copy all state through, output zeros
time >= max_sequence_length, empty_update,
# otherwise calculation is required: copy some or all of it through
_maybe_copy_some_through)
if len(final_output_and_state) != len(flat_zero_output) + len(flat_state):
raise ValueError("Internal error: state and output were not concatenated "
"correctly.")
final_output = final_output_and_state[:len(flat_zero_output)]
final_state = final_output_and_state[len(flat_zero_output):]
for output, flat_output in zip(final_output, flat_zero_output):
output.set_shape(flat_output.get_shape())
for substate, flat_substate in zip(final_state, flat_state):
substate.set_shape(flat_substate.get_shape())
final_output = nest.pack_sequence_as(
structure=zero_output, flat_sequence=final_output)
final_state = nest.pack_sequence_as(
structure=state, flat_sequence=final_state)
return final_output, final_state
def dynamic_rnn(cell, inputs, inerval, sequence_length=None, initial_state=None,
dtype=None, parallel_iterations=None, swap_memory=False,
time_major=False, scope=None):
"""Creates a recurrent neural network specified by RNNCell `cell`.
This function is functionally identical to the function `rnn` above, but
performs fully dynamic unrolling of `inputs`.
Unlike `rnn`, the input `inputs` is not a Python list of `Tensors`, one for
each frame. Instead, `inputs` may be a single `Tensor` where
the maximum time is either the first or second dimension (see the parameter
`time_major`). Alternatively, it may be a (possibly nested) tuple of
Tensors, each of them having matching batch and time dimensions.
The corresponding output is either a single `Tensor` having the same number
of time steps and batch size, or a (possibly nested) tuple of such tensors,
matching the nested structure of `cell.output_size`.
The parameter `sequence_length` is optional and is used to copy-through state
and zero-out outputs when past a batch element's sequence length. So it's more
for correctness than performance, unlike in rnn().
Args:
cell: An instance of RNNCell.
inputs: The RNN inputs.
If `time_major == False` (default), this must be a `Tensor` of shape:
`[batch_size, max_time, ...]`, or a nested tuple of such
elements.
If `time_major == True`, this must be a `Tensor` of shape:
`[max_time, batch_size, ...]`, or a nested tuple of such
elements.
This may also be a (possibly nested) tuple of Tensors satisfying
this property. The first two dimensions must match across all the inputs,
but otherwise the ranks and other shape components may differ.
In this case, input to `cell` at each time-step will replicate the
structure of these tuples, except for the time dimension (from which the
time is taken).
The input to `cell` at each time step will be a `Tensor` or (possibly
nested) tuple of Tensors each with dimensions `[batch_size, ...]`.
sequence_length: (optional) An int32/int64 vector sized `[batch_size]`.
initial_state: (optional) An initial state for the RNN.
If `cell.state_size` is an integer, this must be
a `Tensor` of appropriate type and shape `[batch_size, cell.state_size]`.
If `cell.state_size` is a tuple, this should be a tuple of
tensors having shapes `[batch_size, s] for s in cell.state_size`.
dtype: (optional) The data type for the initial state and expected output.
Required if initial_state is not provided or RNN state has a heterogeneous
dtype.
parallel_iterations: (Default: 32). The number of iterations to run in
parallel. Those operations which do not have any temporal dependency
and can be run in parallel, will be. This parameter trades off
time for space. Values >> 1 use more memory but take less time,
while smaller values use less memory but computations take longer.
swap_memory: Transparently swap the tensors produced in forward inference
but needed for back prop from GPU to CPU. This allows training RNNs
which would typically not fit on a single GPU, with very minimal (or no)
performance penalty.
time_major: The shape format of the `inputs` and `outputs` Tensors.
If true, these `Tensors` must be shaped `[max_time, batch_size, depth]`.
If false, these `Tensors` must be shaped `[batch_size, max_time, depth]`.
Using `time_major = True` is a bit more efficient because it avoids
transposes at the beginning and end of the RNN calculation. However,
most TensorFlow data is batch-major, so by default this function
accepts input and emits output in batch-major form.
scope: VariableScope for the created subgraph; defaults to "rnn".
Returns:
A pair (outputs, state) where:
outputs: The RNN output `Tensor`.
If time_major == False (default), this will be a `Tensor` shaped:
`[batch_size, max_time, cell.output_size]`.
If time_major == True, this will be a `Tensor` shaped:
`[max_time, batch_size, cell.output_size]`.
Note, if `cell.output_size` is a (possibly nested) tuple of integers
or `TensorShape` objects, then `outputs` will be a tuple having the
same structure as `cell.output_size`, containing Tensors having shapes
corresponding to the shape data in `cell.output_size`.
state: The final state. If `cell.state_size` is an int, this
will be shaped `[batch_size, cell.state_size]`. If it is a
`TensorShape`, this will be shaped `[batch_size] + cell.state_size`.
If it is a (possibly nested) tuple of ints or `TensorShape`, this will
be a tuple having the corresponding shapes.
Raises:
TypeError: If `cell` is not an instance of RNNCell.
ValueError: If inputs is None or an empty list.
"""
# pylint: disable=protected-access
if not isinstance(cell, rnn_cell_impl._RNNCell):
raise TypeError("cell must be an instance of RNNCell")
# pylint: enable=protected-access
# By default, time_major==False and inputs are batch-major: shaped
# [batch, time, depth]
# For internal calculations, we transpose to [time, batch, depth]
flat_input = nest.flatten(inputs)
if not time_major:
# (B,T,D) => (T,B,D)
flat_input = tuple(array_ops.transpose(input_, [1, 0, 2])
for input_ in flat_input)
parallel_iterations = parallel_iterations or 32
if sequence_length is not None:
sequence_length = math_ops.to_int32(sequence_length)
if sequence_length.get_shape().ndims not in (None, 1):
raise ValueError(
"sequence_length must be a vector of length batch_size, "
"but saw shape: %s" % sequence_length.get_shape())
sequence_length = array_ops.identity( # Just to find it in the graph.
sequence_length, name="sequence_length")
# Create a new scope in which the caching device is either
# determined by the parent scope, or is set to place the cached
# Variable using the same placement as for the rest of the RNN.
with vs.variable_scope(scope or "rnn") as varscope:
if varscope.caching_device is None:
varscope.set_caching_device(lambda op: op.device)
input_shape = tuple(array_ops.shape(input_) for input_ in flat_input)
batch_size = input_shape[0][1]
for input_ in input_shape:
if input_[1].get_shape() != batch_size.get_shape():
raise ValueError("All inputs should have the same batch size")
if initial_state is not None:
state = initial_state
else:
if not dtype:
raise ValueError("If no initial_state is provided, dtype must be.")
state = cell.zero_state(batch_size, dtype)
def _assert_has_shape(x, shape):
x_shape = array_ops.shape(x)
packed_shape = array_ops.stack(shape)
return control_flow_ops.Assert(
math_ops.reduce_all(math_ops.equal(x_shape, packed_shape)),
["Expected shape for Tensor %s is " % x.name,
packed_shape, " but saw shape: ", x_shape])
if sequence_length is not None:
# Perform some shape validation
with ops.control_dependencies(
[_assert_has_shape(sequence_length, [batch_size])]):
sequence_length = array_ops.identity(
sequence_length, name="CheckSeqLen")
inputs = nest.pack_sequence_as(structure=inputs, flat_sequence=flat_input)
(outputs, final_state) = _dynamic_rnn_loop(
cell,
inputs,
state,
inerval,
parallel_iterations=parallel_iterations,
swap_memory=swap_memory,
sequence_length=sequence_length,
dtype=dtype)
# Outputs of _dynamic_rnn_loop are always shaped [time, batch, depth].
# If we are performing batch-major calculations, transpose output back
# to shape [batch, time, depth]
if not time_major:
# (T,B,D) => (B,T,D)
flat_output = nest.flatten(outputs)
flat_output = [array_ops.transpose(output, [1, 0, 2])
for output in flat_output]
outputs = nest.pack_sequence_as(
structure=outputs, flat_sequence=flat_output)
return (outputs, final_state)
def _dynamic_rnn_loop(cell,
inputs,
initial_state,
interval,
parallel_iterations,
swap_memory,
sequence_length=None,
dtype=None):
"""Internal implementation of Dynamic RNN.
Args:
cell: An instance of RNNCell.
inputs: A `Tensor` of shape [time, batch_size, input_size], or a nested
tuple of such elements.
initial_state: A `Tensor` of shape `[batch_size, state_size]`, or if
`cell.state_size` is a tuple, then this should be a tuple of
tensors having shapes `[batch_size, s] for s in cell.state_size`.
parallel_iterations: Positive Python int.
swap_memory: A Python boolean
sequence_length: (optional) An `int32` `Tensor` of shape [batch_size].
dtype: (optional) Expected dtype of output. If not specified, inferred from
initial_state.
Returns:
Tuple `(final_outputs, final_state)`.
final_outputs:
A `Tensor` of shape `[time, batch_size, cell.output_size]`. If
`cell.output_size` is a (possibly nested) tuple of ints or `TensorShape`
objects, then this returns a (possibly nsted) tuple of Tensors matching
the corresponding shapes.
final_state:
A `Tensor`, or possibly nested tuple of Tensors, matching in length
and shapes to `initial_state`.
Raises:
ValueError: If the input depth cannot be inferred via shape inference
from the inputs.
"""
pre_state = initial_state
state = initial_state
assert isinstance(parallel_iterations, int), "parallel_iterations must be int"
state_size = cell.state_size
flat_input = nest.flatten(inputs)
flat_output_size = nest.flatten(cell.output_size)
# Construct an initial output
input_shape = array_ops.shape(flat_input[0])
time_steps = input_shape[0]
batch_size = input_shape[1]
inputs_got_shape = tuple(input_.get_shape().with_rank_at_least(3)
for input_ in flat_input)
const_time_steps, const_batch_size = inputs_got_shape[0].as_list()[:2]
for shape in inputs_got_shape:
if not shape[2:].is_fully_defined():
raise ValueError(
"Input size (depth of inputs) must be accessible via shape inference,"
" but saw value None.")
got_time_steps = shape[0].value
got_batch_size = shape[1].value
if const_time_steps != got_time_steps:
raise ValueError(
"Time steps is not the same for all the elements in the input in a "
"batch.")
if const_batch_size != got_batch_size:
raise ValueError(
"Batch_size is not the same for all the elements in the input.")
# Prepare dynamic conditional copying of state & output
def _create_zero_arrays(size):
size = _state_size_with_prefix(size, prefix=[batch_size])
return array_ops.zeros(
array_ops.stack(size), _infer_state_dtype(dtype, state))
flat_zero_output = tuple(_create_zero_arrays(output)
for output in flat_output_size)
zero_output = nest.pack_sequence_as(structure=cell.output_size,
flat_sequence=flat_zero_output)
if sequence_length is not None:
min_sequence_length = math_ops.reduce_min(sequence_length)
max_sequence_length = math_ops.reduce_max(sequence_length)
time = array_ops.constant(0, dtype=dtypes.int32, name="time")
with ops.name_scope("dynamic_rnn") as scope:
base_name = scope
def _create_ta(name, dtype):
return tensor_array_ops.TensorArray(dtype=dtype,
size=time_steps,
tensor_array_name=base_name + name)
output_ta = tuple(_create_ta("output_%d" % i,
_infer_state_dtype(dtype, state))
for i in range(len(flat_output_size)))
input_ta = tuple(_create_ta("input_%d" % i, flat_input[0].dtype)
for i in range(len(flat_input)))
input_ta = tuple(ta.unstack(input_)
for ta, input_ in zip(input_ta, flat_input))
def _time_step(time, output_ta_t, state, pre_state):
"""Take a time step of the dynamic RNN.
Args:
time: int32 scalar Tensor.
output_ta_t: List of `TensorArray`s that represent the output.
state: nested tuple of vector tensors that represent the state.
Returns:
The tuple (time + 1, output_ta_t with updated flow, new_state).
"""
input_t = tuple(ta.read(time) for ta in input_ta)
# Restore some shape information
for input_, shape in zip(input_t, inputs_got_shape):
input_.set_shape(shape[1:])
new_pre_state = []
input_state = []
for cstate, pstate in zip(list(pre_state),list(state)):
c1 = tf.cond(tf.logical_and(tf.equal(tf.mod(time,interval),tf.constant(0)),
tf.greater(time, tf.constant(0))), lambda: (cstate.c+pstate.c)/2, lambda: pstate.c)
h1 = tf.cond(tf.logical_and(tf.equal(tf.mod(time,interval),tf.constant(0)),
tf.greater(time, tf.constant(0))), lambda: (cstate.h+pstate.h)/2, lambda: pstate.h)
c2 = tf.cond(tf.logical_and(tf.equal(tf.mod(time,interval),tf.constant(0)),
tf.greater(time, tf.constant(0))), lambda: pstate.c, lambda: cstate.c)
h2 = tf.cond(tf.logical_and(tf.equal(tf.mod(time,interval),tf.constant(0)),
tf.greater(time, tf.constant(0))), lambda: pstate.h, lambda: cstate.h)
new_pre_state.append(tf.contrib.rnn.LSTMStateTuple(c2,h2))
input_state.append(tf.contrib.rnn.LSTMStateTuple(c1,h1))
new_pre_state = tuple(new_pre_state)
input_state = tuple(input_state)
input_t = nest.pack_sequence_as(structure=inputs, flat_sequence=input_t)
call_cell = lambda: cell(input_t, input_state)
if sequence_length is not None:
(output, new_state) = _rnn_step(
time=time,
sequence_length=sequence_length,
min_sequence_length=min_sequence_length,
max_sequence_length=max_sequence_length,
zero_output=zero_output,
state=input_state,
call_cell=call_cell,
state_size=state_size,
skip_conditionals=True)
else:
(output, new_state) = call_cell()
# Pack state if using state tuples
output = nest.flatten(output)
output_ta_t = tuple(
ta.write(time, out) for ta, out in zip(output_ta_t, output))
return (time + 1, output_ta_t, new_state, new_pre_state)
_, output_final_ta, final_state, final_pre_state = control_flow_ops.while_loop(
cond=lambda time, *_: time < time_steps,
body=_time_step,
loop_vars=(time, output_ta, state, pre_state),
parallel_iterations=parallel_iterations,
swap_memory=swap_memory)
# Unpack final output if not using output tuples.
final_outputs = tuple(ta.stack() for ta in output_final_ta)
# Restore some shape information
for output, output_size in zip(final_outputs, flat_output_size):
shape = _state_size_with_prefix(
output_size, prefix=[const_time_steps, const_batch_size])
output.set_shape(shape)
final_outputs = nest.pack_sequence_as(
structure=cell.output_size, flat_sequence=final_outputs)
return (final_outputs, final_state)
================================================
FILE: youtube-8m-zhangteng/train-with-rebuild.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for training Tensorflow models on the YouTube-8M dataset."""
import json
import os
import time
import eval_util
import losses
import frame_level_models
import video_level_models
import readers
import tensorflow as tf
import tensorflow.contrib.slim as slim
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
from tensorflow.python.ops import variables as tf_variables
from mygradients import mygradients, mygradients_full
import utils
FLAGS = flags.FLAGS
if __name__ == "__main__":
# Dataset flags.
flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
"The directory to save the model files in.")
flags.DEFINE_string(
"train_data_pattern", "",
"File glob for the training dataset. If the files refer to Frame Level "
"features (i.e. tensorflow.SequenceExample), then set --reader_type "
"format. The (Sequence)Examples are expected to have 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024", "Length of the feature vectors.")
# Model flags.
flags.DEFINE_bool(
"frame_features", False,
"If set, then --train_data_pattern must be frame-level features. "
"Otherwise, --train_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_bool(
"frame_only", False,
"If set, then --train_data_pattern must be frame-level features. "
"Otherwise, --train_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_bool(
"norm", True,
"If set, then --input_data should be l2-normalized before follow-up processing. "
"Otherwise, --input_data remain unchanged")
flags.DEFINE_string(
"model", "LogisticModel",
"Which architecture to use for the model. Models are defined "
"in models.py.")
flags.DEFINE_bool(
"start_new_model", False,
"If set, this will not resume from a checkpoint and will instead create a"
" new model instance.")
# Training flags.
flags.DEFINE_integer("batch_size", 1024,
"How many examples to process per batch for training.")
flags.DEFINE_integer("stride_size", 3,
"How many frames to skip in frame level models, "
"only used in LstmFramesModel and LstmSoftmaxModel.")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Which loss function to use for training the model.")
flags.DEFINE_float(
"regularization_penalty", 1,
"How much weight to give to the regularization loss (the label loss has "
"a weight of 1).")
flags.DEFINE_float("base_learning_rate", 0.01,
"Which learning rate to start with.")
flags.DEFINE_float("learning_rate_decay", 0.95,
"Learning rate decay factor to be applied every "
"learning_rate_decay_examples.")
flags.DEFINE_float("learning_rate_decay_examples", 4000000,
"Multiply current learning rate by learning_rate_decay "
"every learning_rate_decay_examples.")
flags.DEFINE_integer("num_epochs", 5,
"How many passes to make over the dataset before "
"halting training.")
# Other flags.
flags.DEFINE_integer("num_readers", 8,
"How many threads to use for reading input files.")
flags.DEFINE_string("optimizer", "AdamOptimizer",
"What optimizer class to use.")
flags.DEFINE_string("gradient", None,
"What optimizer class to use.")
flags.DEFINE_float("clip_gradient_norm", 0.1, "Norm to clip gradients to.")
flags.DEFINE_bool(
"log_device_placement", False,
"Whether to write the device on which every op will run into the "
"logs on startup.")
def validate_class_name(flag_value, category, modules, expected_superclass):
"""Checks that the given string matches a class of the expected type.
Args:
flag_value: A string naming the class to instantiate.
category: A string used further describe the class in error messages
(e.g. 'model', 'reader', 'loss').
modules: A list of modules to search for the given class.
expected_superclass: A class that the given class should inherit from.
Raises:
FlagsError: If the given class could not be found or if the first class
found with that name doesn't inherit from the expected superclass.
Returns:
True if a class was found that matches the given constraints.
"""
candidates = [getattr(module, flag_value, None) for module in modules]
for candidate in candidates:
if not candidate:
continue
if not issubclass(candidate, expected_superclass):
raise flags.FlagsError("%s '%s' doesn't inherit from %s." %
(category, flag_value,
expected_superclass.__name__))
return True
raise flags.FlagsError("Unable to find %s '%s'." % (category, flag_value))
def get_input_data_tensors(reader,
data_pattern,
batch_size=1000,
num_epochs=None,
num_readers=1):
"""Creates the section of the graph which reads the training data.
Args:
reader: A class which parses the training data.
data_pattern: A 'glob' style path to the data files.
batch_size: How many examples to process at a time.
num_epochs: How many passes to make over the training data. Set to 'None'
to run indefinitely.
num_readers: How many I/O threads to use.
Returns:
A tuple containing the features tensor, labels tensor, and optionally a
tensor containing the number of frames per video. The exact dimensions
depend on the reader being used.
Raises:
IOError: If no files matching the given pattern were found.
"""
logging.info("Using batch size of " + str(batch_size) + " for training.")
with tf.name_scope("train_input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find training files. data_pattern='" +
data_pattern + "'.")
logging.info("Number of training files: %s.", str(len(files)))
filename_queue = tf.train.string_input_producer(
files, num_epochs=num_epochs, shuffle=True)
training_data = [
reader.prepare_reader(filename_queue) for _ in range(num_readers)
]
return tf.train.shuffle_batch_join(
training_data,
batch_size=batch_size,
capacity=FLAGS.batch_size * 5,
min_after_dequeue=FLAGS.batch_size,
allow_smaller_final_batch=True,
enqueue_many=True)
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def build_graph(reader,
model,
train_data_pattern,
label_loss_fn=losses.CrossEntropyLoss(),
batch_size=1000,
base_learning_rate=0.01,
learning_rate_decay_examples=1000000,
learning_rate_decay=0.95,
optimizer_class=tf.train.AdamOptimizer,
clip_gradient_norm=1.0,
regularization_penalty=1,
num_readers=1,
num_epochs=None):
"""Creates the Tensorflow graph.
This will only be called once in the life of
a training model, because after the graph is created the model will be
restored from a meta graph file rather than being recreated.
Args:
reader: The data file reader. It should inherit from BaseReader.
model: The core model (e.g. logistic or neural net). It should inherit
from BaseModel.
train_data_pattern: glob path to the training data files.
label_loss_fn: What kind of loss to apply to the model. It should inherit
from BaseLoss.
batch_size: How many examples to process at a time.
base_learning_rate: What learning rate to initialize the optimizer with.
optimizer_class: Which optimization algorithm to use.
clip_gradient_norm: Magnitude of the gradient to clip to.
regularization_penalty: How much weight to give the regularization loss
compared to the label loss.
num_readers: How many threads to use for I/O operations.
num_epochs: How many passes to make over the data. 'None' means an
unlimited number of passes.
"""
global_step = tf.Variable(0, trainable=False, name="global_step")
learning_rate = tf.train.exponential_decay(
base_learning_rate,
global_step * batch_size,
learning_rate_decay_examples,
learning_rate_decay,
staircase=True)
tf.summary.scalar('learning_rate', learning_rate)
unused_video_id, model_input_raw, labels_batch, num_frames = (
get_input_data_tensors(
reader,
train_data_pattern,
batch_size=batch_size,
num_readers=num_readers,
num_epochs=num_epochs))
tf.summary.histogram("model/input_raw", model_input_raw)
feature_dim = len(model_input_raw.get_shape()) - 1
if FLAGS.norm:
model_input = tf.nn.l2_normalize(model_input_raw, feature_dim)
else:
model_input = model_input_raw
with tf.name_scope("model"):
result = model.create_model(
model_input,
num_frames=num_frames,
vocab_size=reader.num_classes,
labels=labels_batch)
for variable in slim.get_model_variables():
tf.summary.histogram(variable.op.name, variable)
predictions = result["predictions"]
if "predictions_negative" in result.keys():
predictions_negative = result["predictions_negative"]
else:
predictions_negative = 1-predictions
if "predictions_positive" in result.keys():
predictions_positive = result["predictions_positive"]
else:
predictions_positive = predictions
if predictions.get_shape().ndims==3:
predictions = tf.reshape(predictions,[-1,predictions.get_shape().as_list()[2]])
labels_batch = tf.reshape(labels_batch,[-1,labels_batch.get_shape().as_list()[2]])
if "bottleneck" in result.keys():
bottle_neck = result["bottleneck"]
else:
bottle_neck = tf.constant(0.0)
if "predictions_class" in result.keys():
predictions_class = result["predictions_class"]
else:
predictions_class = predictions
if "predictions_encoder" in result.keys():
predictions_encoder = result["predictions_encoder"]
else:
predictions_encoder = predictions
if "predictions_experts" in result.keys():
predictions_experts = result["predictions_experts"]
else:
predictions_experts = predictions
if "predictions_postprocess" in result.keys():
predictions_postprocess = result["predictions_postprocess"]
else:
predictions_postprocess = predictions
if "loss" in result.keys():
append_loss = result["loss"]
else:
append_loss = tf.constant(0.0)
if "predictions_encoder" in result.keys():
label_loss, float_encoders = label_loss_fn.calculate_loss_mix2(predictions, predictions_class, predictions_encoder, labels_batch)
tf.summary.histogram("model/float_encoders", float_encoders)
elif "predictions_class" in result.keys():
label_loss = label_loss_fn.calculate_loss_mix(predictions, predictions_class, labels_batch)
elif "predictions_experts" in result.keys():
label_loss = label_loss_fn.calculate_loss_max(predictions, predictions_experts, labels_batch)
elif "predictions_postprocess" in result.keys():
label_loss = label_loss_fn.calculate_loss_postprocess(predictions_postprocess, labels_batch)
elif "predictions_negative" in result.keys():
label_loss = label_loss_fn.calculate_loss_negative(predictions_positive, predictions_negative, labels_batch)
else:
label_loss = label_loss_fn.calculate_loss(predictions, labels_batch)
if "prediction_frames" in result.keys():
predictions_frames = result["prediction_frames"]
labels_frames = tf.tile(tf.reshape(labels_batch,[-1,1,reader.num_classes]),[1,FLAGS.moe_num_extend,1])
labels_frames = tf.cast(tf.reshape(labels_frames,[-1,reader.num_classes]),tf.float32)
frame_loss = label_loss_fn.calculate_loss(predictions_frames, labels_frames)
if "prediction_prepare_frames" in result.keys():
prediction_prepare_frames = result["prediction_prepare_frames"]
prediction_prepare_video = result["prediction_prepare_video"]
max_frames = model_input.get_shape().as_list()[1]
frames_sum = tf.reduce_sum(tf.abs(model_input),axis=2)
frames_true = tf.ones(tf.shape(frames_sum))
frames_false = tf.zeros(tf.shape(frames_sum))
frames_bool = tf.where(tf.greater(frames_sum, frames_false), frames_true, frames_false)
frames_bool = tf.reshape(frames_bool[:,0:max_frames:FLAGS.stride_size],[-1,1])
labels_prepare_frames = tf.tile(tf.reshape(labels_batch,[-1,1,reader.num_classes]),[1,max_frames//FLAGS.stride_size,1])
labels_prepare_frames = tf.cast(tf.reshape(labels_prepare_frames,[-1,reader.num_classes]),tf.float32)*frames_bool
prediction_prepare_frames = prediction_prepare_frames*frames_bool
label_loss = 0.1*label_loss_fn.calculate_loss(prediction_prepare_frames, labels_prepare_frames) + \
0.1*label_loss_fn.calculate_loss(prediction_prepare_video, labels_batch)
else:
label_loss = label_loss*0.0
else:
frame_loss = tf.constant(0.0)
tf.summary.scalar("label_loss", label_loss)
if "regularization_loss" in result.keys():
reg_loss = result["regularization_loss"]
else:
reg_loss = tf.constant(0.0)
reg_losses = tf.losses.get_regularization_losses()
if reg_losses:
reg_loss += tf.add_n(reg_losses)
if regularization_penalty != 0:
tf.summary.scalar("reg_loss", reg_loss)
# Adds update_ops (e.g., moving average updates in batch normalization) as
# a dependency to the train_op.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
if "update_ops" in result.keys():
update_ops += result["update_ops"]
if update_ops:
with tf.control_dependencies(update_ops):
barrier = tf.no_op(name="gradient_barrier")
with tf.control_dependencies([barrier]):
label_loss = tf.identity(label_loss)
# Incorporate the L2 weight penalties etc.
final_loss = regularization_penalty * reg_loss + label_loss + frame_loss + append_loss
if FLAGS.gradient=="my":
opt = tf.train.AdamOptimizer(learning_rate=learning_rate)
variables_to_train = tf_variables.trainable_variables()
top_grads, top_vars = mygradients(final_loss, variables_to_train, global_step=global_step, name="mygradients_net")
grads_and_vars = list(zip(top_grads, top_vars))
train_op = opt.apply_gradients(grads_and_vars, global_step=global_step)
else:
optimizer = optimizer_class(learning_rate)
train_op = slim.learning.create_train_op(
final_loss,
optimizer,
global_step=global_step,
clip_gradient_norm=clip_gradient_norm)
tf.add_to_collection("global_step", global_step)
tf.add_to_collection("loss", final_loss)
tf.add_to_collection("reg_loss", reg_loss)
tf.add_to_collection("bottleneck", bottle_neck)
tf.add_to_collection("predictions", predictions)
tf.add_to_collection("input_batch_raw", model_input_raw)
tf.add_to_collection("input_batch", model_input)
tf.add_to_collection("num_frames", num_frames)
tf.add_to_collection("labels", tf.cast(labels_batch, tf.float32))
tf.add_to_collection("train_op", train_op)
class Trainer(object):
"""A Trainer to train a Tensorflow graph."""
def __init__(self, cluster, task, train_dir, log_device_placement=True):
""""Creates a Trainer.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
self.cluster = cluster
self.task = task
self.is_master = (task.type == "master" and task.index == 0)
self.train_dir = train_dir
self.config = tf.ConfigProto(log_device_placement=log_device_placement)
if self.is_master and self.task.index > 0:
raise StandardError("%s: Only one replica of master expected",
task_as_string(self.task))
def run(self, start_new_model=False):
"""Performs training on the currently defined Tensorflow graph.
Returns:
A tuple of the training Hit@1 and the training PERR.
"""
if self.is_master and start_new_model:
self.remove_training_directory(self.train_dir)
target, device_fn = self.start_server_if_distributed()
checkpoint = self.get_latest_checkpoint(start_new_model, self.train_dir)
with tf.Graph().as_default() as graph:
restorer, saver = self.build_model()
global_step = tf.get_collection("global_step")[0]
loss = tf.get_collection("loss")[0]
reg_loss = tf.get_collection("reg_loss")[0]
predictions = tf.get_collection("predictions")[0]
labels = tf.get_collection("labels")[0]
train_op = tf.get_collection("train_op")[0]
init_op = tf.global_variables_initializer()
sv = tf.train.Supervisor(
graph,
logdir=self.train_dir,
init_op=init_op,
is_chief=self.is_master,
global_step=global_step,
save_model_secs=15 * 60,
save_summaries_secs=120,
saver=saver)
logging.info("%s: Starting managed session.", task_as_string(self.task))
with sv.managed_session(target, config=self.config) as sess:
if checkpoint is not None:
restorer.restore(sess, checkpoint)
try:
logging.info("%s: Entering training loop.", task_as_string(self.task))
while not sv.should_stop():
batch_start_time = time.time()
_, global_step_val, loss_val, reg_loss_val, predictions_val, labels_val = sess.run(
[train_op, global_step, loss, reg_loss, predictions, labels])
seconds_per_batch = time.time() - batch_start_time
if self.is_master:
examples_per_second = labels_val.shape[0] / seconds_per_batch
hit_at_one = eval_util.calculate_hit_at_one(predictions_val,
labels_val)
perr = eval_util.calculate_precision_at_equal_recall_rate(
predictions_val, labels_val)
gap = eval_util.calculate_gap(predictions_val, labels_val)
logging.info(
"%s: training step " + str(global_step_val) + "| Hit@1: " +
("%.2f" % hit_at_one) + " PERR: " + ("%.2f" % perr) + " GAP: " +
("%.2f" % gap) + " Loss: " + str(loss_val) +
" RegLoss: " + str(reg_loss_val),
task_as_string(self.task))
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_Hit@1", hit_at_one),
global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_Perr", perr), global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_GAP", gap), global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("global_step/Examples/Second",
examples_per_second), global_step_val)
sv.summary_writer.flush()
except tf.errors.OutOfRangeError:
logging.info("%s: Done training -- epoch limit reached.",
task_as_string(self.task))
logging.info("%s: Exited training loop.", task_as_string(self.task))
sv.Stop()
def start_server_if_distributed(self):
"""Starts a server if the execution is distributed."""
if self.cluster:
logging.info("%s: Starting trainer within cluster %s.",
task_as_string(self.task), self.cluster.as_dict())
server = start_server(self.cluster, self.task)
target = server.target
device_fn = tf.train.replica_device_setter(
ps_device="/job:ps",
worker_device="/job:%s/task:%d" % (self.task.type, self.task.index),
cluster=self.cluster)
else:
target = ""
device_fn = ""
return (target, device_fn)
def remove_training_directory(self, train_dir):
"""Removes the training directory."""
try:
logging.info(
"%s: Removing existing train directory.",
task_as_string(self.task))
gfile.DeleteRecursively(train_dir)
except:
logging.error(
"%s: Failed to delete directory " + train_dir +
" when starting a new model. Please delete it manually and" +
" try again.", task_as_string(self.task))
def get_meta_filename(self, start_new_model, train_dir):
if start_new_model:
logging.info("%s: Flag 'start_new_model' is set. Building a new model.",
task_as_string(self.task))
return None
latest_checkpoint = tf.train.latest_checkpoint(train_dir)
if not latest_checkpoint:
logging.info("%s: No checkpoint file found. Building a new model.",
task_as_string(self.task))
return None
meta_filename = latest_checkpoint + ".meta"
if not gfile.Exists(meta_filename):
logging.info("%s: No meta graph file found. Building a new model.",
task_as_string(self.task))
return None
else:
return meta_filename
def recover_model(self, meta_filename):
logging.info("%s: Restoring from meta graph file %s",
task_as_string(self.task), meta_filename)
return tf.train.import_meta_graph(meta_filename)
def get_latest_checkpoint(self, start_new_model, train_dir):
if start_new_model:
logging.info("%s: Flag 'start_new_model' is set. Building a new model.",
task_as_string(self.task))
return None
latest_checkpoint = tf.train.latest_checkpoint(train_dir)
if not latest_checkpoint:
logging.info("%s: No checkpoint file found. Building a new model.",
task_as_string(self.task))
return None
return latest_checkpoint
def build_model(self):
"""Find the model and build the graph."""
# Convert feature_names and feature_sizes to lists of values.
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
if FLAGS.frame_only:
reader = readers.YT8MFrameFeatureOnlyReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
# Find the model.
model = find_class_by_name(FLAGS.model,
[frame_level_models, video_level_models])()
label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])()
optimizer_class = find_class_by_name(FLAGS.optimizer, [tf.train])
build_graph(reader=reader,
model=model,
optimizer_class=optimizer_class,
clip_gradient_norm=FLAGS.clip_gradient_norm,
train_data_pattern=FLAGS.train_data_pattern,
label_loss_fn=label_loss_fn,
base_learning_rate=FLAGS.base_learning_rate,
learning_rate_decay=FLAGS.learning_rate_decay,
learning_rate_decay_examples=FLAGS.learning_rate_decay_examples,
regularization_penalty=FLAGS.regularization_penalty,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size,
num_epochs=FLAGS.num_epochs)
logging.info("%s: Built graph.", task_as_string(self.task))
all_vars = tf.trainable_variables()
restore_list = [var for var in all_vars if 'notrestore' not in var.name]
return tf.train.Saver(var_list=restore_list), tf.train.Saver(max_to_keep=2, keep_checkpoint_every_n_hours=1)
class ParameterServer(object):
"""A parameter server to serve variables in a distributed execution."""
def __init__(self, cluster, task):
"""Creates a ParameterServer.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
self.cluster = cluster
self.task = task
def run(self):
"""Starts the parameter server."""
logging.info("%s: Starting parameter server within cluster %s.",
task_as_string(self.task), self.cluster.as_dict())
server = start_server(self.cluster, self.task)
server.join()
def start_server(cluster, task):
"""Creates a Server.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
if not task.type:
raise ValueError("%s: The task type must be specified." %
task_as_string(task))
if task.index is None:
raise ValueError("%s: The task index must be specified." %
task_as_string(task))
# Create and start a server.
return tf.train.Server(
tf.train.ClusterSpec(cluster),
protocol="grpc",
job_name=task.type,
task_index=task.index)
def task_as_string(task):
return "/job:%s/task:%s" % (task.type, task.index)
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object,), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.",
task_as_string(task), tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
Trainer(cluster, task, FLAGS.train_dir, FLAGS.log_device_placement).run(
start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-zhangteng/train.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for training Tensorflow models on the YouTube-8M dataset."""
import json
import os
import time
import eval_util
import losses
import frame_level_models
import video_level_models
import readers
import tensorflow as tf
import tensorflow.contrib.slim as slim
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
from tensorflow.python.ops import variables as tf_variables
from mygradients import mygradients, mygradients_full
import utils
FLAGS = flags.FLAGS
if __name__ == "__main__":
# Dataset flags.
flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
"The directory to save the model files in.")
flags.DEFINE_string(
"train_data_pattern", "",
"File glob for the training dataset. If the files refer to Frame Level "
"features (i.e. tensorflow.SequenceExample), then set --reader_type "
"format. The (Sequence)Examples are expected to have 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024", "Length of the feature vectors.")
# Model flags.
flags.DEFINE_bool(
"frame_features", False,
"If set, then --train_data_pattern must be frame-level features. "
"Otherwise, --train_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_bool(
"frame_only", False,
"If set, then --train_data_pattern must be frame-level features. "
"Otherwise, --train_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_bool(
"norm", True,
"If set, then --input_data should be l2-normalized before follow-up processing. "
"Otherwise, --input_data remain unchanged")
flags.DEFINE_string(
"model", "LogisticModel",
"Which architecture to use for the model. Models are defined "
"in models.py.")
flags.DEFINE_bool(
"start_new_model", False,
"If set, this will not resume from a checkpoint and will instead create a"
" new model instance.")
# Distillation flags
flags.DEFINE_bool(
"distillation_features", False,
"If set, *DistillationFeatureReader will be used, the feature must contains"
"prediction features (shape = [4716]).")
flags.DEFINE_integer(
"distillation_type", 2, "Type of distillation, options are 0, 1, 2 and 3.")
flags.DEFINE_float("distillation_percent", 0.5,
"If larger than 0, final_loss = distillation_loss * percent + normal_loss * (1.0 - percent).")
flags.DEFINE_float("gpu", 1.0,
"Gpu memory percent.")
# Training flags.
flags.DEFINE_integer("batch_size", 1024,
"How many examples to process per batch for training.")
flags.DEFINE_integer("stride_size", 3,
"How many frames to skip in frame level models, "
"only used in LstmFramesModel and LstmSoftmaxModel.")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Which loss function to use for training the model.")
flags.DEFINE_float(
"regularization_penalty", 1,
"How much weight to give to the regularization loss (the label loss has "
"a weight of 1).")
flags.DEFINE_float("base_learning_rate", 0.01,
"Which learning rate to start with.")
flags.DEFINE_float("learning_rate_decay", 0.95,
"Learning rate decay factor to be applied every "
"learning_rate_decay_examples.")
flags.DEFINE_float("learning_rate_decay_examples", 4000000,
"Multiply current learning rate by learning_rate_decay "
"every learning_rate_decay_examples.")
flags.DEFINE_integer("num_epochs", 5,
"How many passes to make over the dataset before "
"halting training.")
# Other flags.
flags.DEFINE_integer("num_readers", 8,
"How many threads to use for reading input files.")
flags.DEFINE_string("optimizer", "AdamOptimizer",
"What optimizer class to use.")
flags.DEFINE_string("gradient", None,
"What optimizer class to use.")
flags.DEFINE_float("clip_gradient_norm", 0.1, "Norm to clip gradients to.")
flags.DEFINE_bool(
"log_device_placement", False,
"Whether to write the device on which every op will run into the "
"logs on startup.")
def validate_class_name(flag_value, category, modules, expected_superclass):
"""Checks that the given string matches a class of the expected type.
Args:
flag_value: A string naming the class to instantiate.
category: A string used further describe the class in error messages
(e.g. 'model', 'reader', 'loss').
modules: A list of modules to search for the given class.
expected_superclass: A class that the given class should inherit from.
Raises:
FlagsError: If the given class could not be found or if the first class
found with that name doesn't inherit from the expected superclass.
Returns:
True if a class was found that matches the given constraints.
"""
candidates = [getattr(module, flag_value, None) for module in modules]
for candidate in candidates:
if not candidate:
continue
if not issubclass(candidate, expected_superclass):
raise flags.FlagsError("%s '%s' doesn't inherit from %s." %
(category, flag_value,
expected_superclass.__name__))
return True
raise flags.FlagsError("Unable to find %s '%s'." % (category, flag_value))
def get_input_data_tensors(reader,
data_pattern,
batch_size=1000,
num_epochs=None,
num_readers=1):
"""Creates the section of the graph which reads the training data.
Args:
reader: A class which parses the training data.
data_pattern: A 'glob' style path to the data files.
batch_size: How many examples to process at a time.
num_epochs: How many passes to make over the training data. Set to 'None'
to run indefinitely.
num_readers: How many I/O threads to use.
Returns:
A tuple containing the features tensor, labels tensor, and optionally a
tensor containing the number of frames per video. The exact dimensions
depend on the reader being used.
Raises:
IOError: If no files matching the given pattern were found.
"""
logging.info("Using batch size of " + str(batch_size) + " for training.")
with tf.name_scope("train_input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find training files. data_pattern='" +
data_pattern + "'.")
logging.info("Number of training files: %s.", str(len(files)))
filename_queue = tf.train.string_input_producer(
files, num_epochs=num_epochs, shuffle=True)
training_data = [
reader.prepare_reader(filename_queue) for _ in range(num_readers)
]
return tf.train.shuffle_batch_join(
training_data,
batch_size=batch_size,
capacity=FLAGS.batch_size * 5,
min_after_dequeue=FLAGS.batch_size,
allow_smaller_final_batch=True,
enqueue_many=True)
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def build_graph(reader,
model,
train_data_pattern,
label_loss_fn=losses.CrossEntropyLoss(),
batch_size=1000,
base_learning_rate=0.01,
learning_rate_decay_examples=1000000,
learning_rate_decay=0.95,
optimizer_class=tf.train.AdamOptimizer,
clip_gradient_norm=1.0,
regularization_penalty=1,
num_readers=1,
num_epochs=None):
"""Creates the Tensorflow graph.
This will only be called once in the life of
a training model, because after the graph is created the model will be
restored from a meta graph file rather than being recreated.
Args:
reader: The data file reader. It should inherit from BaseReader.
model: The core model (e.g. logistic or neural net). It should inherit
from BaseModel.
train_data_pattern: glob path to the training data files.
label_loss_fn: What kind of loss to apply to the model. It should inherit
from BaseLoss.
batch_size: How many examples to process at a time.
base_learning_rate: What learning rate to initialize the optimizer with.
optimizer_class: Which optimization algorithm to use.
clip_gradient_norm: Magnitude of the gradient to clip to.
regularization_penalty: How much weight to give the regularization loss
compared to the label loss.
num_readers: How many threads to use for I/O operations.
num_epochs: How many passes to make over the data. 'None' means an
unlimited number of passes.
"""
global_step = tf.Variable(0, trainable=False, name="global_step")
learning_rate = tf.train.exponential_decay(
base_learning_rate,
global_step * batch_size,
learning_rate_decay_examples,
learning_rate_decay,
staircase=True)
tf.summary.scalar('learning_rate', learning_rate)
if FLAGS.distillation_features:
unused_video_id, model_input_raw, labels_batch, num_frames, distill_labels_batch = (
get_input_data_tensors(
reader,
train_data_pattern,
batch_size=batch_size,
num_readers=num_readers,
num_epochs=num_epochs))
if FLAGS.distillation_features and FLAGS.distillation_type == 2:
p = FLAGS.distillation_percent
print("distillation_percent =", p, "reforming labels")
float_labels = tf.cast(labels_batch, dtype=tf.float32)
sum_float_labels = tf.reduce_sum(float_labels, axis=1, keep_dims=True)
sum_distill_labels = tf.reduce_sum(distill_labels_batch, axis=1, keep_dims=True) + 1e-6
distill_labels_batch = float_labels + distill_labels_batch * (sum_float_labels / sum_distill_labels * p)
distill_labels_batch = tf.clip_by_value(distill_labels_batch, clip_value_min=0.0, clip_value_max=1.0)
else:
unused_video_id, model_input_raw, labels_batch, num_frames = (
get_input_data_tensors(
reader,
train_data_pattern,
batch_size=batch_size,
num_readers=num_readers,
num_epochs=num_epochs))
tf.summary.histogram("model/input_raw", model_input_raw)
feature_dim = len(model_input_raw.get_shape()) - 1
if FLAGS.norm:
model_input = tf.nn.l2_normalize(model_input_raw, feature_dim)
else:
model_input = model_input_raw
with tf.name_scope("model"):
if FLAGS.distillation_features and FLAGS.distillation_type == 0:
result = model.create_model(
model_input,
num_frames=num_frames,
vocab_size=reader.num_classes,
labels=labels_batch,
distill_labels=distill_labels_batch)
else:
result = model.create_model(
model_input,
num_frames=num_frames,
vocab_size=reader.num_classes,
labels=labels_batch)
for variable in slim.get_model_variables():
tf.summary.histogram(variable.op.name, variable)
predictions = result["predictions"]
if "predictions_negative" in result.keys():
predictions_negative = result["predictions_negative"]
else:
predictions_negative = 1-predictions
if "predictions_positive" in result.keys():
predictions_positive = result["predictions_positive"]
else:
predictions_positive = predictions
if predictions.get_shape().ndims==3:
predictions = tf.reshape(predictions,[-1,predictions.get_shape().as_list()[2]])
labels_batch = tf.reshape(labels_batch,[-1,labels_batch.get_shape().as_list()[2]])
if "bottleneck" in result.keys():
bottle_neck = result["bottleneck"]
else:
bottle_neck = tf.constant(0.0)
if "predictions_class" in result.keys():
predictions_class = result["predictions_class"]
else:
predictions_class = predictions
if "predictions_encoder" in result.keys():
predictions_encoder = result["predictions_encoder"]
else:
predictions_encoder = predictions
if "predictions_experts" in result.keys():
predictions_experts = result["predictions_experts"]
else:
predictions_experts = predictions
if "predictions_postprocess" in result.keys():
predictions_postprocess = result["predictions_postprocess"]
else:
predictions_postprocess = predictions
if "loss" in result.keys():
append_loss = result["loss"]
else:
append_loss = tf.constant(0.0)
if "predictions_encoder" in result.keys():
label_loss, float_encoders = label_loss_fn.calculate_loss_mix2(predictions, predictions_class, predictions_encoder, labels_batch)
tf.summary.histogram("model/float_encoders", float_encoders)
elif FLAGS.distillation_features and FLAGS.distillation_type == 1:
label_loss = label_loss_fn.calculate_loss_distill_boost(predictions, distill_labels_batch, labels_batch)
elif FLAGS.distillation_features and FLAGS.distillation_type == 2:
label_loss = label_loss_fn.calculate_loss_distill(predictions, distill_labels_batch, labels_batch)
elif FLAGS.distillation_features and FLAGS.distillation_type == 3:
label_loss = label_loss_fn.calculate_loss_distill_relabel(predictions, distill_labels_batch, labels_batch)
elif "predictions_class" in result.keys():
label_loss = label_loss_fn.calculate_loss_mix(predictions, predictions_class, labels_batch)
elif "predictions_experts" in result.keys():
label_loss = label_loss_fn.calculate_loss_max(predictions, predictions_experts, labels_batch)
elif "predictions_postprocess" in result.keys():
label_loss = label_loss_fn.calculate_loss_postprocess(predictions_postprocess, labels_batch)
elif "predictions_negative" in result.keys():
label_loss = label_loss_fn.calculate_loss_negative(predictions_positive, predictions_negative, labels_batch)
else:
label_loss = label_loss_fn.calculate_loss(predictions, labels_batch)
if "prediction_frames" in result.keys():
predictions_frames = result["prediction_frames"]
labels_frames = tf.tile(tf.reshape(labels_batch,[-1,1,reader.num_classes]),[1,FLAGS.moe_num_extend,1])
labels_frames = tf.cast(tf.reshape(labels_frames,[-1,reader.num_classes]),tf.float32)
frame_loss = label_loss_fn.calculate_loss(predictions_frames, labels_frames)
if "prediction_prepare_frames" in result.keys():
prediction_prepare_frames = result["prediction_prepare_frames"]
prediction_prepare_video = result["prediction_prepare_video"]
max_frames = model_input.get_shape().as_list()[1]
frames_sum = tf.reduce_sum(tf.abs(model_input),axis=2)
frames_true = tf.ones(tf.shape(frames_sum))
frames_false = tf.zeros(tf.shape(frames_sum))
frames_bool = tf.where(tf.greater(frames_sum, frames_false), frames_true, frames_false)
frames_bool = tf.reshape(frames_bool[:,0:max_frames:FLAGS.stride_size],[-1,1])
labels_prepare_frames = tf.tile(tf.reshape(labels_batch,[-1,1,reader.num_classes]),[1,max_frames//FLAGS.stride_size,1])
labels_prepare_frames = tf.cast(tf.reshape(labels_prepare_frames,[-1,reader.num_classes]),tf.float32)*frames_bool
prediction_prepare_frames = prediction_prepare_frames*frames_bool
label_loss = 0.1*label_loss_fn.calculate_loss(prediction_prepare_frames, labels_prepare_frames) + \
0.1*label_loss_fn.calculate_loss(prediction_prepare_video, labels_batch)
else:
label_loss = label_loss*0.0
elif "prediction_minmax" in result.keys():
predictions_minmax = result["prediction_minmax"]
predictions_min = tf.reduce_min(predictions_minmax,axis=1)
predictions_max = tf.reduce_max(predictions_minmax,axis=1)
epsilon = 10e-6
float_labels = tf.cast(labels_batch, tf.float32)
cross_entropy_loss = float_labels * tf.log(predictions_min + epsilon) + (
1 - float_labels) * tf.log(1 - predictions_max + epsilon)
frame_loss = tf.reduce_mean(tf.reduce_sum(tf.negative(cross_entropy_loss), 1))
label_loss = label_loss*0.0
else:
frame_loss = tf.constant(0.0)
tf.summary.scalar("label_loss", label_loss)
if "regularization_loss" in result.keys():
reg_loss = result["regularization_loss"]
else:
reg_loss = tf.constant(0.0)
reg_losses = tf.losses.get_regularization_losses()
if reg_losses:
reg_loss += tf.add_n(reg_losses)
if regularization_penalty != 0:
tf.summary.scalar("reg_loss", reg_loss)
# Adds update_ops (e.g., moving average updates in batch normalization) as
# a dependency to the train_op.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
if "update_ops" in result.keys():
update_ops += result["update_ops"]
if update_ops:
with tf.control_dependencies(update_ops):
barrier = tf.no_op(name="gradient_barrier")
with tf.control_dependencies([barrier]):
label_loss = tf.identity(label_loss)
# Incorporate the L2 weight penalties etc.
final_loss = regularization_penalty * reg_loss + label_loss + frame_loss + append_loss
if FLAGS.gradient=="my":
opt = tf.train.AdamOptimizer(learning_rate=learning_rate)
variables_to_train = tf_variables.trainable_variables()
top_grads, top_vars = mygradients(final_loss, variables_to_train, global_step=global_step, name="mygradients_net")
grads_and_vars = list(zip(top_grads, top_vars))
train_op = opt.apply_gradients(grads_and_vars, global_step=global_step)
else:
optimizer = optimizer_class(learning_rate)
train_op = slim.learning.create_train_op(
final_loss,
optimizer,
global_step=global_step,
clip_gradient_norm=clip_gradient_norm)
tf.add_to_collection("global_step", global_step)
tf.add_to_collection("loss", final_loss)
tf.add_to_collection("reg_loss", reg_loss)
tf.add_to_collection("bottleneck", bottle_neck)
tf.add_to_collection("predictions", predictions)
tf.add_to_collection("input_batch_raw", model_input_raw)
tf.add_to_collection("input_batch", model_input)
tf.add_to_collection("num_frames", num_frames)
tf.add_to_collection("labels", tf.cast(labels_batch, tf.float32))
tf.add_to_collection("train_op", train_op)
class Trainer(object):
"""A Trainer to train a Tensorflow graph."""
def __init__(self, cluster, task, train_dir, log_device_placement=True):
""""Creates a Trainer.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
self.cluster = cluster
self.task = task
self.is_master = (task.type == "master" and task.index == 0)
self.train_dir = train_dir
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=FLAGS.gpu)
self.config = tf.ConfigProto(log_device_placement=log_device_placement)
if self.is_master and self.task.index > 0:
raise StandardError("%s: Only one replica of master expected",
task_as_string(self.task))
def run(self, start_new_model=False):
"""Performs training on the currently defined Tensorflow graph.
Returns:
A tuple of the training Hit@1 and the training PERR.
"""
if self.is_master and start_new_model:
self.remove_training_directory(self.train_dir)
target, device_fn = self.start_server_if_distributed()
meta_filename = self.get_meta_filename(start_new_model, self.train_dir)
with tf.Graph().as_default() as graph:
if meta_filename:
saver = self.recover_model(meta_filename)
with tf.device(device_fn):
if not meta_filename:
saver = self.build_model()
global_step = tf.get_collection("global_step")[0]
loss = tf.get_collection("loss")[0]
reg_loss = tf.get_collection("reg_loss")[0]
predictions = tf.get_collection("predictions")[0]
labels = tf.get_collection("labels")[0]
train_op = tf.get_collection("train_op")[0]
init_op = tf.global_variables_initializer()
sv = tf.train.Supervisor(
graph,
logdir=self.train_dir,
init_op=init_op,
is_chief=self.is_master,
global_step=global_step,
save_model_secs=15 * 60,
save_summaries_secs=120,
saver=saver)
logging.info("%s: Starting managed session.", task_as_string(self.task))
with sv.managed_session(target, config=self.config) as sess:
try:
logging.info("%s: Entering training loop.", task_as_string(self.task))
while not sv.should_stop():
batch_start_time = time.time()
_, global_step_val, loss_val, reg_loss_val, predictions_val, labels_val = sess.run(
[train_op, global_step, loss, reg_loss, predictions, labels])
seconds_per_batch = time.time() - batch_start_time
if self.is_master:
examples_per_second = labels_val.shape[0] / seconds_per_batch
hit_at_one = eval_util.calculate_hit_at_one(predictions_val,
labels_val)
perr = eval_util.calculate_precision_at_equal_recall_rate(
predictions_val, labels_val)
gap = eval_util.calculate_gap(predictions_val, labels_val)
logging.info(
"%s: training step " + str(global_step_val) + "| Hit@1: " +
("%.2f" % hit_at_one) + " PERR: " + ("%.2f" % perr) + " GAP: " +
("%.2f" % gap) + " Loss: " + str(loss_val) +
" RegLoss: " + str(reg_loss_val),
task_as_string(self.task))
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_Hit@1", hit_at_one),
global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_Perr", perr), global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_GAP", gap), global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("global_step/Examples/Second",
examples_per_second), global_step_val)
sv.summary_writer.flush()
except tf.errors.OutOfRangeError:
logging.info("%s: Done training -- epoch limit reached.",
task_as_string(self.task))
logging.info("%s: Exited training loop.", task_as_string(self.task))
sv.Stop()
def start_server_if_distributed(self):
"""Starts a server if the execution is distributed."""
if self.cluster:
logging.info("%s: Starting trainer within cluster %s.",
task_as_string(self.task), self.cluster.as_dict())
server = start_server(self.cluster, self.task)
target = server.target
device_fn = tf.train.replica_device_setter(
ps_device="/job:ps",
worker_device="/job:%s/task:%d" % (self.task.type, self.task.index),
cluster=self.cluster)
else:
target = ""
device_fn = ""
return (target, device_fn)
def remove_training_directory(self, train_dir):
"""Removes the training directory."""
try:
logging.info(
"%s: Removing existing train directory.",
task_as_string(self.task))
gfile.DeleteRecursively(train_dir)
except:
logging.error(
"%s: Failed to delete directory " + train_dir +
" when starting a new model. Please delete it manually and" +
" try again.", task_as_string(self.task))
def get_meta_filename(self, start_new_model, train_dir):
if start_new_model:
logging.info("%s: Flag 'start_new_model' is set. Building a new model.",
task_as_string(self.task))
return None
latest_checkpoint = tf.train.latest_checkpoint(train_dir)
if not latest_checkpoint:
logging.info("%s: No checkpoint file found. Building a new model.",
task_as_string(self.task))
return None
meta_filename = latest_checkpoint + ".meta"
if not gfile.Exists(meta_filename):
logging.info("%s: No meta graph file found. Building a new model.",
task_as_string(self.task))
return None
else:
return meta_filename
def recover_model(self, meta_filename):
logging.info("%s: Restoring from meta graph file %s",
task_as_string(self.task), meta_filename)
return tf.train.import_meta_graph(meta_filename)
def build_model(self):
"""Find the model and build the graph."""
# Convert feature_names and feature_sizes to lists of values.
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.distillation_features:
print("distillation readers")
if FLAGS.frame_features:
reader = readers.YT8MFrameDistillationFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedDistillationFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
if FLAGS.frame_features:
if FLAGS.frame_only:
reader = readers.YT8MFrameFeatureOnlyReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
# Find the model.
model = find_class_by_name(FLAGS.model,
[frame_level_models, video_level_models])()
label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])()
optimizer_class = find_class_by_name(FLAGS.optimizer, [tf.train])
build_graph(reader=reader,
model=model,
optimizer_class=optimizer_class,
clip_gradient_norm=FLAGS.clip_gradient_norm,
train_data_pattern=FLAGS.train_data_pattern,
label_loss_fn=label_loss_fn,
base_learning_rate=FLAGS.base_learning_rate,
learning_rate_decay=FLAGS.learning_rate_decay,
learning_rate_decay_examples=FLAGS.learning_rate_decay_examples,
regularization_penalty=FLAGS.regularization_penalty,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size,
num_epochs=FLAGS.num_epochs)
logging.info("%s: Built graph.", task_as_string(self.task))
return tf.train.Saver(max_to_keep=2, keep_checkpoint_every_n_hours=1)
class ParameterServer(object):
"""A parameter server to serve variables in a distributed execution."""
def __init__(self, cluster, task):
"""Creates a ParameterServer.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
self.cluster = cluster
self.task = task
def run(self):
"""Starts the parameter server."""
logging.info("%s: Starting parameter server within cluster %s.",
task_as_string(self.task), self.cluster.as_dict())
server = start_server(self.cluster, self.task)
server.join()
def start_server(cluster, task):
"""Creates a Server.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
if not task.type:
raise ValueError("%s: The task type must be specified." %
task_as_string(task))
if task.index is None:
raise ValueError("%s: The task index must be specified." %
task_as_string(task))
# Create and start a server.
return tf.train.Server(
tf.train.ClusterSpec(cluster),
protocol="grpc",
job_name=task.type,
task_index=task.index)
def task_as_string(task):
return "/job:%s/task:%s" % (task.type, task.index)
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object,), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.",
task_as_string(task), tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
Trainer(cluster, task, FLAGS.train_dir, FLAGS.log_device_placement).run(
start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-zhangteng/train_autoencoder.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for training Tensorflow models on the YouTube-8M dataset."""
import json
import os
import time
import eval_util
import losses
import labels_autoencoder
import readers
import numpy as np
import tensorflow as tf
import tensorflow.contrib.slim as slim
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import utils
FLAGS = flags.FLAGS
if __name__ == "__main__":
# Dataset flags.
flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
"The directory to save the model files in.")
flags.DEFINE_string(
"train_data_pattern", "",
"File glob for the training dataset. If the files refer to Frame Level "
"features (i.e. tensorflow.SequenceExample), then set --reader_type "
"format. The (Sequence)Examples are expected to have 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024", "Length of the feature vectors.")
# Model flags.
flags.DEFINE_bool(
"frame_features", False,
"If set, then --train_data_pattern must be frame-level features. "
"Otherwise, --train_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_bool(
"frame_only", False,
"If set, then --train_data_pattern must be frame-level features. "
"Otherwise, --train_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_string(
"model", "LogisticModel",
"Which architecture to use for the model. Models are defined "
"in models.py.")
flags.DEFINE_bool(
"start_new_model", False,
"If set, this will not resume from a checkpoint and will instead create a"
" new model instance.")
# Training flags.
flags.DEFINE_integer("batch_size", 1024,
"How many examples to process per batch for training.")
flags.DEFINE_integer("stride_size", 4,
"How many examples to process per batch for training.")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Which loss function to use for training the model.")
flags.DEFINE_float(
"regularization_penalty", 1,
"How much weight to give to the regularization loss (the label loss has "
"a weight of 1).")
flags.DEFINE_float("base_learning_rate", 0.001,
"Which learning rate to start with.")
flags.DEFINE_float("learning_rate_decay", 0.95,
"Learning rate decay factor to be applied every "
"learning_rate_decay_examples.")
flags.DEFINE_float("learning_rate_decay_examples", 4000000,
"Multiply current learning rate by learning_rate_decay "
"every learning_rate_decay_examples.")
flags.DEFINE_integer("num_epochs", 5,
"How many passes to make over the dataset before "
"halting training.")
# Other flags.
flags.DEFINE_integer("num_readers", 8,
"How many threads to use for reading input files.")
flags.DEFINE_string("optimizer", "AdamOptimizer",
"What optimizer class to use.")
flags.DEFINE_float("clip_gradient_norm", 0.1, "Norm to clip gradients to.")
flags.DEFINE_bool(
"log_device_placement", False,
"Whether to write the device on which every op will run into the "
"logs on startup.")
def validate_class_name(flag_value, category, modules, expected_superclass):
"""Checks that the given string matches a class of the expected type.
Args:
flag_value: A string naming the class to instantiate.
category: A string used further describe the class in error messages
(e.g. 'model', 'reader', 'loss').
modules: A list of modules to search for the given class.
expected_superclass: A class that the given class should inherit from.
Raises:
FlagsError: If the given class could not be found or if the first class
found with that name doesn't inherit from the expected superclass.
Returns:
True if a class was found that matches the given constraints.
"""
candidates = [getattr(module, flag_value, None) for module in modules]
for candidate in candidates:
if not candidate:
continue
if not issubclass(candidate, expected_superclass):
raise flags.FlagsError("%s '%s' doesn't inherit from %s." %
(category, flag_value,
expected_superclass.__name__))
return True
raise flags.FlagsError("Unable to find %s '%s'." % (category, flag_value))
def get_input_data_tensors(reader,
data_pattern,
batch_size=1000,
num_epochs=None,
num_readers=1):
"""Creates the section of the graph which reads the training data.
Args:
reader: A class which parses the training data.
data_pattern: A 'glob' style path to the data files.
batch_size: How many examples to process at a time.
num_epochs: How many passes to make over the training data. Set to 'None'
to run indefinitely.
num_readers: How many I/O threads to use.
Returns:
A tuple containing the features tensor, labels tensor, and optionally a
tensor containing the number of frames per video. The exact dimensions
depend on the reader being used.
Raises:
IOError: If no files matching the given pattern were found.
"""
logging.info("Using batch size of " + str(batch_size) + " for training.")
with tf.name_scope("train_input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find training files. data_pattern='" +
data_pattern + "'.")
logging.info("Number of training files: %s.", str(len(files)))
filename_queue = tf.train.string_input_producer(
files, num_epochs=num_epochs, shuffle=True)
training_data = [
reader.prepare_reader(filename_queue) for _ in range(num_readers)
]
return tf.train.shuffle_batch_join(
training_data,
batch_size=batch_size,
capacity=FLAGS.batch_size * 5,
min_after_dequeue=FLAGS.batch_size,
allow_smaller_final_batch=True,
enqueue_many=True)
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def get_forward_parameters(vocab_size=4716):
t_vars = tf.trainable_variables()
h1_vars_weight = [var for var in t_vars if 'hidden_1' in var.name and 'weights' in var.name]
h1_vars_biases = [var for var in t_vars if 'hidden_1' in var.name and 'biases' in var.name]
h2_vars_weight = [var for var in t_vars if 'hidden_2' in var.name and 'weights' in var.name]
h2_vars_biases = [var for var in t_vars if 'hidden_2' in var.name and 'biases' in var.name]
o1_vars_weight = [var for var in t_vars if 'output_1' in var.name and 'weights' in var.name]
o1_vars_biases = [var for var in t_vars if 'output_1' in var.name and 'biases' in var.name]
o2_vars_weight = [var for var in t_vars if 'output_2' in var.name and 'weights' in var.name]
o2_vars_biases = [var for var in t_vars if 'output_2' in var.name and 'biases' in var.name]
h1_vars_biases = tf.reshape(h1_vars_biases[0],[1,FLAGS.hidden_size_1])
h2_vars_biases = tf.reshape(h2_vars_biases[0],[1,FLAGS.hidden_size_2])
o1_vars_biases = tf.reshape(o1_vars_biases[0],[1,FLAGS.hidden_size_1])
o2_vars_biases = tf.reshape(o2_vars_biases[0],[1,vocab_size])
vars_1 = tf.concat((h1_vars_weight[0],h1_vars_biases),axis=0)
vars_2 = tf.concat((h2_vars_weight[0],h2_vars_biases),axis=0)
vars_3 = tf.concat((o1_vars_weight[0],o1_vars_biases),axis=0)
vars_4 = tf.concat((o2_vars_weight[0],o2_vars_biases),axis=0)
return [vars_1,vars_2,vars_3,vars_4]
def build_graph(reader,
model,
train_data_pattern,
label_loss_fn=losses.CrossEntropyLoss(),
batch_size=1000,
base_learning_rate=0.01,
learning_rate_decay_examples=1000000,
learning_rate_decay=0.95,
optimizer_class=tf.train.AdamOptimizer,
clip_gradient_norm=1.0,
regularization_penalty=1,
num_readers=1,
num_epochs=None):
"""Creates the Tensorflow graph.
This will only be called once in the life of
a training model, because after the graph is created the model will be
restored from a meta graph file rather than being recreated.
Args:
reader: The data file reader. It should inherit from BaseReader.
model: The core model (e.g. logistic or neural net). It should inherit
from BaseModel.
train_data_pattern: glob path to the training data files.
label_loss_fn: What kind of loss to apply to the model. It should inherit
from BaseLoss.
batch_size: How many examples to process at a time.
base_learning_rate: What learning rate to initialize the optimizer with.
optimizer_class: Which optimization algorithm to use.
clip_gradient_norm: Magnitude of the gradient to clip to.
regularization_penalty: How much weight to give the regularization loss
compared to the label loss.
num_readers: How many threads to use for I/O operations.
num_epochs: How many passes to make over the data. 'None' means an
unlimited number of passes.
"""
global_step = tf.Variable(0, trainable=False, name="global_step")
learning_rate = tf.train.exponential_decay(
base_learning_rate,
global_step * batch_size,
learning_rate_decay_examples,
learning_rate_decay,
staircase=True)
tf.summary.scalar('learning_rate', learning_rate)
optimizer = optimizer_class(learning_rate)
unused_video_id, model_input_raw, labels_batch, num_frames = (
get_input_data_tensors(
reader,
train_data_pattern,
batch_size=batch_size,
num_readers=num_readers,
num_epochs=num_epochs))
tf.summary.histogram("model/input_raw", model_input_raw)
with tf.name_scope("model"):
result = model.create_model(
labels_batch,
num_frames=num_frames,
vocab_size=reader.num_classes,
labels=labels_batch)
parameters = get_forward_parameters(vocab_size=reader.num_classes)
for variable in slim.get_model_variables():
tf.summary.histogram(variable.op.name, variable)
predictions = result["predictions"]
if predictions.get_shape().ndims==3:
predictions = tf.reshape(predictions,[-1,predictions.get_shape().as_list()[2]])
labels_batch = tf.reshape(labels_batch,[-1,labels_batch.get_shape().as_list()[2]])
if "bottleneck" in result.keys():
bottle_neck = result["bottleneck"]
else:
bottle_neck = tf.constant(0.0)
if "predictions_class" in result.keys():
predictions_class = result["predictions_class"]
else:
predictions_class = predictions
if "loss_sparse" in result.keys():
sparse_loss = result["loss_sparse"]
else:
sparse_loss = tf.constant(0.0)
if "loss" in result.keys():
label_loss = result["loss"]
elif "predictions_class" in result.keys():
label_loss = label_loss_fn.calculate_loss_mix(predictions, predictions_class, labels_batch)
else:
label_loss = label_loss_fn.calculate_loss(predictions, labels_batch)
tf.summary.scalar("label_loss", label_loss)
if "regularization_loss" in result.keys():
reg_loss = result["regularization_loss"]
else:
reg_loss = tf.constant(0.0)
reg_losses = tf.losses.get_regularization_losses()
if reg_losses:
reg_loss += tf.add_n(reg_losses)
if regularization_penalty != 0:
tf.summary.scalar("reg_loss", reg_loss)
# Adds update_ops (e.g., moving average updates in batch normalization) as
# a dependency to the train_op.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
if "update_ops" in result.keys():
update_ops += result["update_ops"]
if update_ops:
with tf.control_dependencies(update_ops):
barrier = tf.no_op(name="gradient_barrier")
with tf.control_dependencies([barrier]):
label_loss = tf.identity(label_loss)
# Incorporate the L2 weight penalties etc.
final_loss = regularization_penalty * reg_loss + label_loss + sparse_loss
train_op = slim.learning.create_train_op(
final_loss,
optimizer,
global_step=global_step,
clip_gradient_norm=clip_gradient_norm)
tf.add_to_collection("global_step", global_step)
tf.add_to_collection("loss", final_loss)
tf.add_to_collection("reg_loss", reg_loss)
tf.add_to_collection("bottleneck", bottle_neck)
tf.add_to_collection("predictions", predictions)
tf.add_to_collection("input_batch_raw", model_input_raw)
tf.add_to_collection("num_frames", num_frames)
tf.add_to_collection("labels", tf.cast(labels_batch, tf.float32))
tf.add_to_collection("train_op", train_op)
tf.add_to_collection("parameters", parameters)
class Trainer(object):
"""A Trainer to train a Tensorflow graph."""
def __init__(self, cluster, task, train_dir, log_device_placement=True):
""""Creates a Trainer.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
self.cluster = cluster
self.task = task
self.is_master = (task.type == "master" and task.index == 0)
self.train_dir = train_dir
self.config = tf.ConfigProto(log_device_placement=log_device_placement)
if self.is_master and self.task.index > 0:
raise StandardError("%s: Only one replica of master expected",
task_as_string(self.task))
def run(self, start_new_model=False):
"""Performs training on the currently defined Tensorflow graph.
Returns:
A tuple of the training Hit@1 and the training PERR.
"""
if self.is_master and start_new_model:
self.remove_training_directory(self.train_dir)
target, device_fn = self.start_server_if_distributed()
meta_filename = self.get_meta_filename(start_new_model, self.train_dir)
with tf.Graph().as_default() as graph:
if meta_filename:
saver = self.recover_model(meta_filename)
with tf.device(device_fn):
if not meta_filename:
saver = self.build_model()
global_step = tf.get_collection("global_step")[0]
loss = tf.get_collection("loss")[0]
reg_loss = tf.get_collection("reg_loss")[0]
predictions = tf.get_collection("predictions")[0]
labels = tf.get_collection("labels")[0]
train_op = tf.get_collection("train_op")[0]
parameters = tf.get_collection("parameters")
init_op = tf.global_variables_initializer()
sv = tf.train.Supervisor(
graph,
logdir=self.train_dir,
init_op=init_op,
is_chief=self.is_master,
global_step=global_step,
save_model_secs=15 * 60,
save_summaries_secs=120,
saver=saver)
logging.info("%s: Starting managed session.", task_as_string(self.task))
with sv.managed_session(target, config=self.config) as sess:
try:
logging.info("%s: Entering training loop.", task_as_string(self.task))
while not sv.should_stop():
batch_start_time = time.time()
_, global_step_val, loss_val, reg_loss_val, predictions_val, labels_val = sess.run(
[train_op, global_step, loss, reg_loss, predictions, labels])
seconds_per_batch = time.time() - batch_start_time
if self.is_master:
examples_per_second = labels_val.shape[0] / seconds_per_batch
hit_at_one = eval_util.calculate_hit_at_one(predictions_val,
labels_val)
perr = eval_util.calculate_precision_at_equal_recall_rate(
predictions_val, labels_val)
gap = eval_util.calculate_gap(predictions_val, labels_val)
logging.info(
"%s: training step " + str(global_step_val) + "| Hit@1: " +
("%.2f" % hit_at_one) + " PERR: " + ("%.2f" % perr) + " GAP: " +
("%.2f" % gap) + " Loss: " + str(loss_val) +
" RegLoss: " + str(reg_loss_val),
task_as_string(self.task))
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_Hit@1", hit_at_one),
global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_Perr", perr), global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_GAP", gap), global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("global_step/Examples/Second",
examples_per_second), global_step_val)
sv.summary_writer.flush()
params = sess.run(parameters)
for i in range(len(params)):
np.savetxt(FLAGS.train_dir+'autoencoder_layer%d.model' % i,params[i])
except tf.errors.OutOfRangeError:
logging.info("%s: Done training -- epoch limit reached.",
task_as_string(self.task))
logging.info("%s: Exited training loop.", task_as_string(self.task))
sv.Stop()
def start_server_if_distributed(self):
"""Starts a server if the execution is distributed."""
if self.cluster:
logging.info("%s: Starting trainer within cluster %s.",
task_as_string(self.task), self.cluster.as_dict())
server = start_server(self.cluster, self.task)
target = server.target
device_fn = tf.train.replica_device_setter(
ps_device="/job:ps",
worker_device="/job:%s/task:%d" % (self.task.type, self.task.index),
cluster=self.cluster)
else:
target = ""
device_fn = ""
return (target, device_fn)
def remove_training_directory(self, train_dir):
"""Removes the training directory."""
try:
logging.info(
"%s: Removing existing train directory.",
task_as_string(self.task))
gfile.DeleteRecursively(train_dir)
except:
logging.error(
"%s: Failed to delete directory " + train_dir +
" when starting a new model. Please delete it manually and" +
" try again.", task_as_string(self.task))
def get_meta_filename(self, start_new_model, train_dir):
if start_new_model:
logging.info("%s: Flag 'start_new_model' is set. Building a new model.",
task_as_string(self.task))
return None
latest_checkpoint = tf.train.latest_checkpoint(train_dir)
if not latest_checkpoint:
logging.info("%s: No checkpoint file found. Building a new model.",
task_as_string(self.task))
return None
meta_filename = latest_checkpoint + ".meta"
if not gfile.Exists(meta_filename):
logging.info("%s: No meta graph file found. Building a new model.",
task_as_string(self.task))
return None
else:
return meta_filename
def recover_model(self, meta_filename):
logging.info("%s: Restoring from meta graph file %s",
task_as_string(self.task), meta_filename)
return tf.train.import_meta_graph(meta_filename)
def build_model(self):
"""Find the model and build the graph."""
# Convert feature_names and feature_sizes to lists of values.
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
if FLAGS.frame_only:
reader = readers.YT8MFrameFeatureOnlyReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
# Find the model.
model = find_class_by_name(FLAGS.model,
[labels_autoencoder])()
label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])()
optimizer_class = find_class_by_name(FLAGS.optimizer, [tf.train])
build_graph(reader=reader,
model=model,
optimizer_class=optimizer_class,
clip_gradient_norm=FLAGS.clip_gradient_norm,
train_data_pattern=FLAGS.train_data_pattern,
label_loss_fn=label_loss_fn,
base_learning_rate=FLAGS.base_learning_rate,
learning_rate_decay=FLAGS.learning_rate_decay,
learning_rate_decay_examples=FLAGS.learning_rate_decay_examples,
regularization_penalty=FLAGS.regularization_penalty,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size,
num_epochs=FLAGS.num_epochs)
logging.info("%s: Built graph.", task_as_string(self.task))
return tf.train.Saver(max_to_keep=2, keep_checkpoint_every_n_hours=0.25)
class ParameterServer(object):
"""A parameter server to serve variables in a distributed execution."""
def __init__(self, cluster, task):
"""Creates a ParameterServer.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
self.cluster = cluster
self.task = task
def run(self):
"""Starts the parameter server."""
logging.info("%s: Starting parameter server within cluster %s.",
task_as_string(self.task), self.cluster.as_dict())
server = start_server(self.cluster, self.task)
server.join()
def start_server(cluster, task):
"""Creates a Server.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
if not task.type:
raise ValueError("%s: The task type must be specified." %
task_as_string(task))
if task.index is None:
raise ValueError("%s: The task index must be specified." %
task_as_string(task))
# Create and start a server.
return tf.train.Server(
tf.train.ClusterSpec(cluster),
protocol="grpc",
job_name=task.type,
task_index=task.index)
def task_as_string(task):
return "/job:%s/task:%s" % (task.type, task.index)
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object,), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.",
task_as_string(task), tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
Trainer(cluster, task, FLAGS.train_dir, FLAGS.log_device_placement).run(
start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-zhangteng/train_embedding.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for training Tensorflow models on the YouTube-8M dataset."""
import json
import os
import time
import eval_util
import losses_embedding
import labels_embedding
import readers
import numpy as np
import tensorflow as tf
import tensorflow.contrib.slim as slim
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
import utils
FLAGS = flags.FLAGS
if __name__ == "__main__":
# Dataset flags.
flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
"The directory to save the model files in.")
flags.DEFINE_string(
"train_data_pattern", "",
"File glob for the training dataset. If the files refer to Frame Level "
"features (i.e. tensorflow.SequenceExample), then set --reader_type "
"format. The (Sequence)Examples are expected to have 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024", "Length of the feature vectors.")
# Model flags.
flags.DEFINE_bool(
"frame_features", False,
"If set, then --train_data_pattern must be frame-level features. "
"Otherwise, --train_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_bool(
"frame_only", False,
"If set, then --train_data_pattern must be frame-level features. "
"Otherwise, --train_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_string(
"model", "LogisticModel",
"Which architecture to use for the model. Models are defined "
"in models.py.")
flags.DEFINE_bool(
"start_new_model", False,
"If set, this will not resume from a checkpoint and will instead create a"
" new model instance.")
# Training flags.
flags.DEFINE_integer("batch_size", 1024,
"How many examples to process per batch for training.")
flags.DEFINE_integer("stride_size", 4,
"How many frames to skip in frame level models, "
"only used in LstmFramesModel and LstmSoftmaxModel.")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Which loss function to use for training the model.")
flags.DEFINE_float(
"regularization_penalty", 1,
"How much weight to give to the regularization loss (the label loss has "
"a weight of 1).")
flags.DEFINE_float("base_learning_rate", 0.001,
"Which learning rate to start with.")
flags.DEFINE_float("learning_rate_decay", 0.95,
"Learning rate decay factor to be applied every "
"learning_rate_decay_examples.")
flags.DEFINE_float("learning_rate_decay_examples", 4000000,
"Multiply current learning rate by learning_rate_decay "
"every learning_rate_decay_examples.")
flags.DEFINE_integer("num_epochs", 5,
"How many passes to make over the dataset before "
"halting training.")
# Other flags.
flags.DEFINE_integer("num_readers", 8,
"How many threads to use for reading input files.")
flags.DEFINE_string("optimizer", "AdamOptimizer",
"What optimizer class to use.")
flags.DEFINE_float("clip_gradient_norm", 0.1, "Norm to clip gradients to.")
flags.DEFINE_bool(
"log_device_placement", False,
"Whether to write the device on which every op will run into the "
"logs on startup.")
def validate_class_name(flag_value, category, modules, expected_superclass):
"""Checks that the given string matches a class of the expected type.
Args:
flag_value: A string naming the class to instantiate.
category: A string used further describe the class in error messages
(e.g. 'model', 'reader', 'loss').
modules: A list of modules to search for the given class.
expected_superclass: A class that the given class should inherit from.
Raises:
FlagsError: If the given class could not be found or if the first class
found with that name doesn't inherit from the expected superclass.
Returns:
True if a class was found that matches the given constraints.
"""
candidates = [getattr(module, flag_value, None) for module in modules]
for candidate in candidates:
if not candidate:
continue
if not issubclass(candidate, expected_superclass):
raise flags.FlagsError("%s '%s' doesn't inherit from %s." %
(category, flag_value,
expected_superclass.__name__))
return True
raise flags.FlagsError("Unable to find %s '%s'." % (category, flag_value))
def get_input_data_tensors(reader,
data_pattern,
batch_size=1000,
num_epochs=None,
num_readers=1):
"""Creates the section of the graph which reads the training data.
Args:
reader: A class which parses the training data.
data_pattern: A 'glob' style path to the data files.
batch_size: How many examples to process at a time.
num_epochs: How many passes to make over the training data. Set to 'None'
to run indefinitely.
num_readers: How many I/O threads to use.
Returns:
A tuple containing the features tensor, labels tensor, and optionally a
tensor containing the number of frames per video. The exact dimensions
depend on the reader being used.
Raises:
IOError: If no files matching the given pattern were found.
"""
logging.info("Using batch size of " + str(batch_size) + " for training.")
with tf.name_scope("train_input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find training files. data_pattern='" +
data_pattern + "'.")
logging.info("Number of training files: %s.", str(len(files)))
filename_queue = tf.train.string_input_producer(
files, num_epochs=num_epochs, shuffle=True)
training_data = [
reader.prepare_reader(filename_queue) for _ in range(num_readers)
]
return tf.train.shuffle_batch_join(
training_data,
batch_size=batch_size,
capacity=FLAGS.batch_size * 5,
min_after_dequeue=FLAGS.batch_size,
allow_smaller_final_batch=True,
enqueue_many=True)
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def build_graph(reader,
model,
train_data_pattern,
label_loss_fn=losses_embedding.CrossEntropyLoss(),
batch_size=1000,
base_learning_rate=0.01,
learning_rate_decay_examples=1000000,
learning_rate_decay=0.95,
optimizer_class=tf.train.AdamOptimizer,
clip_gradient_norm=1.0,
regularization_penalty=1,
num_readers=1,
num_epochs=None):
"""Creates the Tensorflow graph.
This will only be called once in the life of
a training model, because after the graph is created the model will be
restored from a meta graph file rather than being recreated.
Args:
reader: The data file reader. It should inherit from BaseReader.
model: The core model (e.g. logistic or neural net). It should inherit
from BaseModel.
train_data_pattern: glob path to the training data files.
label_loss_fn: What kind of loss to apply to the model. It should inherit
from BaseLoss.
batch_size: How many examples to process at a time.
base_learning_rate: What learning rate to initialize the optimizer with.
optimizer_class: Which optimization algorithm to use.
clip_gradient_norm: Magnitude of the gradient to clip to.
regularization_penalty: How much weight to give the regularization loss
compared to the label loss.
num_readers: How many threads to use for I/O operations.
num_epochs: How many passes to make over the data. 'None' means an
unlimited number of passes.
"""
global_step = tf.Variable(0, trainable=False, name="global_step")
learning_rate = tf.train.exponential_decay(
base_learning_rate,
global_step * batch_size,
learning_rate_decay_examples,
learning_rate_decay,
staircase=True)
tf.summary.scalar('learning_rate', learning_rate)
optimizer = optimizer_class(learning_rate)
unused_video_id, model_input_raw, labels_batch, num_frames = (
get_input_data_tensors(
reader,
train_data_pattern,
batch_size=batch_size,
num_readers=num_readers,
num_epochs=num_epochs))
tf.summary.histogram("model/input_raw", model_input_raw)
with tf.name_scope("model"):
result = model.create_model(
labels_batch,
num_frames=num_frames,
vocab_size=reader.num_classes,
labels=labels_batch)
for variable in slim.get_model_variables():
tf.summary.histogram(variable.op.name, variable)
predictions = result["predictions"]
predictions_val = result["predictions_val"]
if predictions.get_shape().ndims==3:
predictions = tf.reshape(predictions,[-1,predictions.get_shape().as_list()[2]])
labels_batch = tf.reshape(labels_batch,[-1,labels_batch.get_shape().as_list()[2]])
if "predictions_class" in result.keys():
predictions_class = result["predictions_class"]
else:
predictions_class = predictions
if "loss_sparse" in result.keys():
sparse_loss = result["loss_sparse"]
else:
sparse_loss = tf.constant(0.0)
if "loss" in result.keys():
label_loss = result["loss"]
elif "predictions_class" in result.keys():
label_loss = label_loss_fn.calculate_loss_mix(predictions, predictions_class, labels_batch)
else:
label_loss = label_loss_fn.calculate_loss(predictions, labels_batch)
tf.summary.scalar("label_loss", label_loss)
if "regularization_loss" in result.keys():
reg_loss = result["regularization_loss"]
else:
reg_loss = tf.constant(0.0)
reg_losses = tf.losses.get_regularization_losses()
if reg_losses:
reg_loss += tf.add_n(reg_losses)
if regularization_penalty != 0:
tf.summary.scalar("reg_loss", reg_loss)
# Adds update_ops (e.g., moving average updates in batch normalization) as
# a dependency to the train_op.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
if "update_ops" in result.keys():
update_ops += result["update_ops"]
if update_ops:
with tf.control_dependencies(update_ops):
barrier = tf.no_op(name="gradient_barrier")
with tf.control_dependencies([barrier]):
label_loss = tf.identity(label_loss)
# Incorporate the L2 weight penalties etc.
final_loss = regularization_penalty * reg_loss + label_loss + sparse_loss
train_op = slim.learning.create_train_op(
final_loss,
optimizer,
global_step=global_step,
clip_gradient_norm=clip_gradient_norm)
tf.add_to_collection("global_step", global_step)
tf.add_to_collection("loss", final_loss)
tf.add_to_collection("reg_loss", reg_loss)
tf.add_to_collection("predictions", predictions)
tf.add_to_collection("predictions_val", predictions_val)
tf.add_to_collection("input_batch_raw", model_input_raw)
tf.add_to_collection("num_frames", num_frames)
tf.add_to_collection("labels", tf.cast(labels_batch, tf.float32))
tf.add_to_collection("train_op", train_op)
class Trainer(object):
"""A Trainer to train a Tensorflow graph."""
def __init__(self, cluster, task, train_dir, log_device_placement=True):
""""Creates a Trainer.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
self.cluster = cluster
self.task = task
self.is_master = (task.type == "master" and task.index == 0)
self.train_dir = train_dir
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.2)
self.config = tf.ConfigProto(log_device_placement=log_device_placement,gpu_options=gpu_options)
if self.is_master and self.task.index > 0:
raise StandardError("%s: Only one replica of master expected",
task_as_string(self.task))
def run(self, start_new_model=False):
"""Performs training on the currently defined Tensorflow graph.
Returns:
A tuple of the training Hit@1 and the training PERR.
"""
if self.is_master and start_new_model:
self.remove_training_directory(self.train_dir)
target, device_fn = self.start_server_if_distributed()
meta_filename = self.get_meta_filename(start_new_model, self.train_dir)
with tf.Graph().as_default() as graph:
if meta_filename:
saver = self.recover_model(meta_filename)
with tf.device(device_fn):
if not meta_filename:
saver = self.build_model()
global_step = tf.get_collection("global_step")[0]
loss = tf.get_collection("loss")[0]
reg_loss = tf.get_collection("reg_loss")[0]
predictions = tf.get_collection("predictions")[0]
predictions_v = tf.get_collection("predictions_val")[0]
labels = tf.get_collection("labels")[0]
train_op = tf.get_collection("train_op")[0]
init_op = tf.global_variables_initializer()
sv = tf.train.Supervisor(
graph,
logdir=self.train_dir,
init_op=init_op,
is_chief=self.is_master,
global_step=global_step,
save_model_secs=15 * 60,
save_summaries_secs=120,
saver=saver)
logging.info("%s: Starting managed session.", task_as_string(self.task))
with sv.managed_session(target, config=self.config) as sess:
try:
logging.info("%s: Entering training loop.", task_as_string(self.task))
while not sv.should_stop():
batch_start_time = time.time()
_, global_step_val, loss_val, reg_loss_val, predictions_val, labels_val = sess.run(
[train_op, global_step, loss, reg_loss, predictions_v, labels])
seconds_per_batch = time.time() - batch_start_time
if self.is_master:
examples_per_second = labels_val.shape[0] / seconds_per_batch
hit_at_one = eval_util.calculate_hit_at_one(predictions_val,
labels_val)
perr = eval_util.calculate_precision_at_equal_recall_rate(
predictions_val, labels_val)
gap = eval_util.calculate_gap(predictions_val, labels_val)
logging.info(
"%s: training step " + str(global_step_val) + "| Hit@1: " +
("%.2f" % hit_at_one) + " PERR: " + ("%.2f" % perr) + " GAP: " +
("%.2f" % gap) + " Loss: " + str(loss_val) +
" RegLoss: " + str(reg_loss_val),
task_as_string(self.task))
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_Hit@1", hit_at_one),
global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_Perr", perr), global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_GAP", gap), global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("global_step/Examples/Second",
examples_per_second), global_step_val)
sv.summary_writer.flush()
except tf.errors.OutOfRangeError:
logging.info("%s: Done training -- epoch limit reached.",
task_as_string(self.task))
logging.info("%s: Exited training loop.", task_as_string(self.task))
sv.Stop()
def start_server_if_distributed(self):
"""Starts a server if the execution is distributed."""
if self.cluster:
logging.info("%s: Starting trainer within cluster %s.",
task_as_string(self.task), self.cluster.as_dict())
server = start_server(self.cluster, self.task)
target = server.target
device_fn = tf.train.replica_device_setter(
ps_device="/job:ps",
worker_device="/job:%s/task:%d" % (self.task.type, self.task.index),
cluster=self.cluster)
else:
target = ""
device_fn = ""
return (target, device_fn)
def remove_training_directory(self, train_dir):
"""Removes the training directory."""
try:
logging.info(
"%s: Removing existing train directory.",
task_as_string(self.task))
gfile.DeleteRecursively(train_dir)
except:
logging.error(
"%s: Failed to delete directory " + train_dir +
" when starting a new model. Please delete it manually and" +
" try again.", task_as_string(self.task))
def get_meta_filename(self, start_new_model, train_dir):
if start_new_model:
logging.info("%s: Flag 'start_new_model' is set. Building a new model.",
task_as_string(self.task))
return None
latest_checkpoint = tf.train.latest_checkpoint(train_dir)
if not latest_checkpoint:
logging.info("%s: No checkpoint file found. Building a new model.",
task_as_string(self.task))
return None
meta_filename = latest_checkpoint + ".meta"
if not gfile.Exists(meta_filename):
logging.info("%s: No meta graph file found. Building a new model.",
task_as_string(self.task))
return None
else:
return meta_filename
def recover_model(self, meta_filename):
logging.info("%s: Restoring from meta graph file %s",
task_as_string(self.task), meta_filename)
return tf.train.import_meta_graph(meta_filename)
def build_model(self):
"""Find the model and build the graph."""
# Convert feature_names and feature_sizes to lists of values.
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
if FLAGS.frame_only:
reader = readers.YT8MFrameFeatureOnlyReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
# Find the model.
model = find_class_by_name(FLAGS.model,
[labels_embedding])()
label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses_embedding])()
optimizer_class = find_class_by_name(FLAGS.optimizer, [tf.train])
build_graph(reader=reader,
model=model,
optimizer_class=optimizer_class,
clip_gradient_norm=FLAGS.clip_gradient_norm,
train_data_pattern=FLAGS.train_data_pattern,
label_loss_fn=label_loss_fn,
base_learning_rate=FLAGS.base_learning_rate,
learning_rate_decay=FLAGS.learning_rate_decay,
learning_rate_decay_examples=FLAGS.learning_rate_decay_examples,
regularization_penalty=FLAGS.regularization_penalty,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size,
num_epochs=FLAGS.num_epochs)
logging.info("%s: Built graph.", task_as_string(self.task))
return tf.train.Saver(max_to_keep=2, keep_checkpoint_every_n_hours=0.25)
class ParameterServer(object):
"""A parameter server to serve variables in a distributed execution."""
def __init__(self, cluster, task):
"""Creates a ParameterServer.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
self.cluster = cluster
self.task = task
def run(self):
"""Starts the parameter server."""
logging.info("%s: Starting parameter server within cluster %s.",
task_as_string(self.task), self.cluster.as_dict())
server = start_server(self.cluster, self.task)
server.join()
def start_server(cluster, task):
"""Creates a Server.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
if not task.type:
raise ValueError("%s: The task type must be specified." %
task_as_string(task))
if task.index is None:
raise ValueError("%s: The task index must be specified." %
task_as_string(task))
# Create and start a server.
return tf.train.Server(
tf.train.ClusterSpec(cluster),
protocol="grpc",
job_name=task.type,
task_index=task.index)
def task_as_string(task):
return "/job:%s/task:%s" % (task.type, task.index)
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object,), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.",
task_as_string(task), tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
Trainer(cluster, task, FLAGS.train_dir, FLAGS.log_device_placement).run(
start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-zhangteng/train_ensemble.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for training Tensorflow models on the YouTube-8M dataset."""
import json
import os
import time
import eval_util
import losses
import frame_level_models
import video_level_models
import readers
import tensorflow as tf
import tensorflow.contrib.slim as slim
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging
from tensorflow.python.ops import variables as tf_variables
from mygradients import mygradients, mygradients_full
import utils
FLAGS = flags.FLAGS
if __name__ == "__main__":
# Dataset flags.
flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
"The directory to save the model files in.")
flags.DEFINE_string(
"train_data_pattern", "",
"File glob for the training dataset. If the files refer to Frame Level "
"features (i.e. tensorflow.SequenceExample), then set --reader_type "
"format. The (Sequence)Examples are expected to have 'rgb' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
"to use for training.")
flags.DEFINE_string("feature_sizes", "1024", "Length of the feature vectors.")
# Model flags.
flags.DEFINE_bool(
"frame_features", False,
"If set, then --train_data_pattern must be frame-level features. "
"Otherwise, --train_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_bool(
"frame_only", False,
"If set, then --train_data_pattern must be frame-level features. "
"Otherwise, --train_data_pattern must be aggregated video-level "
"features. The model must also be set appropriately (i.e. to read 3D "
"batches VS 4D batches.")
flags.DEFINE_bool(
"norm", True,
"If set, then --input_data should be l2-normalized before follow-up processing. "
"Otherwise, --input_data remain unchanged")
flags.DEFINE_string(
"model", "LogisticModel",
"Which architecture to use for the model. Models are defined "
"in models.py.")
flags.DEFINE_bool(
"start_new_model", False,
"If set, this will not resume from a checkpoint and will instead create a"
" new model instance.")
# Distillation flags
flags.DEFINE_string(
"distill_data_pattern", "",
"File glob defining the distill dataset in tensorflow.SequenceExample "
"format. The SequenceExamples are expected to have an 'predictions' byte array "
"sequence feature as well as a 'labels' int64 context feature.")
flags.DEFINE_string("distill_names", "predictions", "Name of the distill feature "
"to use for eval.")
flags.DEFINE_string("distill_sizes", "4716", "Length of the distill feature vectors.")
flags.DEFINE_bool(
"distillation_features", False,
"If set, *DistillationFeatureReader will be used, the feature must contains"
"prediction features (shape = [4716]).")
flags.DEFINE_integer(
"distillation_type", 2, "Type of distillation, options are 0, 1, 2 and 3.")
flags.DEFINE_float("distillation_percent", 0.5,
"If larger than 0, final_loss = distillation_loss * percent + normal_loss * (1.0 - percent).")
flags.DEFINE_float("gpu", 1.0,
"Gpu memory percent.")
# Training flags.
flags.DEFINE_integer("batch_size", 1024,
"How many examples to process per batch for training.")
flags.DEFINE_integer("stride_size", 3,
"How many frames to skip in frame level models, "
"only used in LstmFramesModel and LstmSoftmaxModel.")
flags.DEFINE_string("label_loss", "CrossEntropyLoss",
"Which loss function to use for training the model.")
flags.DEFINE_float(
"regularization_penalty", 1,
"How much weight to give to the regularization loss (the label loss has "
"a weight of 1).")
flags.DEFINE_float("base_learning_rate", 0.01,
"Which learning rate to start with.")
flags.DEFINE_float("learning_rate_decay", 0.95,
"Learning rate decay factor to be applied every "
"learning_rate_decay_examples.")
flags.DEFINE_float("learning_rate_decay_examples", 4000000,
"Multiply current learning rate by learning_rate_decay "
"every learning_rate_decay_examples.")
flags.DEFINE_integer("num_epochs", 5,
"How many passes to make over the dataset before "
"halting training.")
# Other flags.
flags.DEFINE_integer("num_readers", 8,
"How many threads to use for reading input files.")
flags.DEFINE_string("optimizer", "AdamOptimizer",
"What optimizer class to use.")
flags.DEFINE_string("gradient", None,
"What optimizer class to use.")
flags.DEFINE_float("clip_gradient_norm", 0.1, "Norm to clip gradients to.")
flags.DEFINE_bool(
"log_device_placement", False,
"Whether to write the device on which every op will run into the "
"logs on startup.")
def validate_class_name(flag_value, category, modules, expected_superclass):
"""Checks that the given string matches a class of the expected type.
Args:
flag_value: A string naming the class to instantiate.
category: A string used further describe the class in error messages
(e.g. 'model', 'reader', 'loss').
modules: A list of modules to search for the given class.
expected_superclass: A class that the given class should inherit from.
Raises:
FlagsError: If the given class could not be found or if the first class
found with that name doesn't inherit from the expected superclass.
Returns:
True if a class was found that matches the given constraints.
"""
candidates = [getattr(module, flag_value, None) for module in modules]
for candidate in candidates:
if not candidate:
continue
if not issubclass(candidate, expected_superclass):
raise flags.FlagsError("%s '%s' doesn't inherit from %s." %
(category, flag_value,
expected_superclass.__name__))
return True
raise flags.FlagsError("Unable to find %s '%s'." % (category, flag_value))
def get_input_data_tensors(reader,
data_pattern,
batch_size=1024,
num_epochs=None,
num_readers=1):
"""Creates the section of the graph which reads the evaluation data.
Args:
reader: A class which parses the training data.
data_pattern: A 'glob' style path to the data files.
batch_size: How many examples to process at a time.
num_readers: How many I/O threads to use.
Returns:
A tuple containing the features tensor, labels tensor, and optionally a
tensor containing the number of frames per video. The exact dimensions
depend on the reader being used.
Raises:
IOError: If no files matching the given pattern were found.
"""
logging.info("Using batch size of " + str(batch_size) + " for evaluation.")
with tf.name_scope("eval_input"):
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find the evaluation files.")
logging.info("number of evaluation files: " + str(len(files)))
files.sort()
filename_queue = tf.train.string_input_producer(
files, shuffle=False, num_epochs=num_epochs)
training_data = reader.prepare_reader(filename_queue)
return tf.train.batch(
training_data,
batch_size=batch_size,
capacity=5 * FLAGS.batch_size,
allow_smaller_final_batch=True,
enqueue_many=True)
def find_class_by_name(name, modules):
"""Searches the provided modules for the named class and returns it."""
modules = [getattr(module, name, None) for module in modules]
return next(a for a in modules if a)
def build_graph(reader1,
reader2,
model,
train_data_pattern,
distill_data_pattern,
label_loss_fn=losses.CrossEntropyLoss(),
batch_size=1000,
base_learning_rate=0.01,
learning_rate_decay_examples=1000000,
learning_rate_decay=0.95,
optimizer_class=tf.train.AdamOptimizer,
clip_gradient_norm=1.0,
regularization_penalty=1,
num_readers=1,
num_epochs=None):
"""Creates the Tensorflow graph.
This will only be called once in the life of
a training model, because after the graph is created the model will be
restored from a meta graph file rather than being recreated.
Args:
reader: The data file reader. It should inherit from BaseReader.
model: The core model (e.g. logistic or neural net). It should inherit
from BaseModel.
train_data_pattern: glob path to the training data files.
label_loss_fn: What kind of loss to apply to the model. It should inherit
from BaseLoss.
batch_size: How many examples to process at a time.
base_learning_rate: What learning rate to initialize the optimizer with.
optimizer_class: Which optimization algorithm to use.
clip_gradient_norm: Magnitude of the gradient to clip to.
regularization_penalty: How much weight to give the regularization loss
compared to the label loss.
num_readers: How many threads to use for I/O operations.
num_epochs: How many passes to make over the data. 'None' means an
unlimited number of passes.
"""
global_step = tf.Variable(0, trainable=False, name="global_step")
learning_rate = tf.train.exponential_decay(
base_learning_rate,
global_step * batch_size,
learning_rate_decay_examples,
learning_rate_decay,
staircase=True)
tf.summary.scalar('learning_rate', learning_rate)
unused_video_id, model_input_raw, labels_batch, num_frames = get_input_data_tensors( # pylint: disable=g-line-too-long
reader1,
train_data_pattern,
batch_size=batch_size,
num_readers=num_readers,
num_epochs=num_epochs)
unused_video_id, distill_labels_batch, unused_labels_batch, unusednum_frames = get_input_data_tensors( # pylint: disable=g-line-too-long
reader2,
distill_data_pattern,
batch_size=batch_size,
num_readers=num_readers,
num_epochs=num_epochs)
tf.summary.histogram("model/input_raw", model_input_raw)
feature_dim = len(model_input_raw.get_shape()) - 1
if FLAGS.norm:
model_input = tf.nn.l2_normalize(model_input_raw, feature_dim)
else:
model_input = model_input_raw
with tf.name_scope("model"):
result = model.create_model(
model_input,
num_frames=num_frames,
vocab_size=reader1.num_classes,
labels=labels_batch,
distill_labels=distill_labels_batch)
for variable in slim.get_model_variables():
tf.summary.histogram(variable.op.name, variable)
predictions = result["predictions"]
if "predictions_negative" in result.keys():
predictions_negative = result["predictions_negative"]
else:
predictions_negative = 1-predictions
if "predictions_positive" in result.keys():
predictions_positive = result["predictions_positive"]
else:
predictions_positive = predictions
if predictions.get_shape().ndims==3:
predictions = tf.reshape(predictions,[-1,predictions.get_shape().as_list()[2]])
labels_batch = tf.reshape(labels_batch,[-1,labels_batch.get_shape().as_list()[2]])
if "bottleneck" in result.keys():
bottle_neck = result["bottleneck"]
else:
bottle_neck = tf.constant(0.0)
if "predictions_class" in result.keys():
predictions_class = result["predictions_class"]
else:
predictions_class = predictions
if "predictions_encoder" in result.keys():
predictions_encoder = result["predictions_encoder"]
else:
predictions_encoder = predictions
if "predictions_experts" in result.keys():
predictions_experts = result["predictions_experts"]
else:
predictions_experts = predictions
if "predictions_postprocess" in result.keys():
predictions_postprocess = result["predictions_postprocess"]
else:
predictions_postprocess = predictions
if "loss" in result.keys():
append_loss = result["loss"]
else:
append_loss = tf.constant(0.0)
if "predictions_encoder" in result.keys():
label_loss, float_encoders = label_loss_fn.calculate_loss_mix2(predictions, predictions_class, predictions_encoder, labels_batch)
tf.summary.histogram("model/float_encoders", float_encoders)
elif FLAGS.distillation_features and FLAGS.distillation_type == 1:
label_loss = label_loss_fn.calculate_loss_distill_boost(predictions, distill_labels_batch, labels_batch)
elif FLAGS.distillation_features and FLAGS.distillation_type == 2:
label_loss = label_loss_fn.calculate_loss_distill(predictions, distill_labels_batch, labels_batch)
elif FLAGS.distillation_features and FLAGS.distillation_type == 3:
label_loss = label_loss_fn.calculate_loss_distill_relabel(predictions, distill_labels_batch, labels_batch)
elif "predictions_class" in result.keys():
label_loss = label_loss_fn.calculate_loss_mix(predictions, predictions_class, labels_batch)
elif "predictions_experts" in result.keys():
label_loss = label_loss_fn.calculate_loss_max(predictions, predictions_experts, labels_batch)
elif "predictions_postprocess" in result.keys():
label_loss = label_loss_fn.calculate_loss_postprocess(predictions_postprocess, labels_batch)
elif "predictions_negative" in result.keys():
label_loss = label_loss_fn.calculate_loss_negative(predictions_positive, predictions_negative, labels_batch)
else:
label_loss = label_loss_fn.calculate_loss(predictions, labels_batch)
if "prediction_frames" in result.keys():
predictions_frames = result["prediction_frames"]
labels_frames = tf.tile(tf.reshape(labels_batch,[-1,1,reader1.num_classes]),[1,FLAGS.moe_num_extend,1])
labels_frames = tf.cast(tf.reshape(labels_frames,[-1,reader1.num_classes]),tf.float32)
frame_loss = label_loss_fn.calculate_loss(predictions_frames, labels_frames)
if "prediction_prepare_frames" in result.keys():
prediction_prepare_frames = result["prediction_prepare_frames"]
prediction_prepare_video = result["prediction_prepare_video"]
max_frames = model_input.get_shape().as_list()[1]
frames_sum = tf.reduce_sum(tf.abs(model_input),axis=2)
frames_true = tf.ones(tf.shape(frames_sum))
frames_false = tf.zeros(tf.shape(frames_sum))
frames_bool = tf.where(tf.greater(frames_sum, frames_false), frames_true, frames_false)
frames_bool = tf.reshape(frames_bool[:,0:max_frames:FLAGS.stride_size],[-1,1])
labels_prepare_frames = tf.tile(tf.reshape(labels_batch,[-1,1,reader1.num_classes]),[1,max_frames//FLAGS.stride_size,1])
labels_prepare_frames = tf.cast(tf.reshape(labels_prepare_frames,[-1,reader1.num_classes]),tf.float32)*frames_bool
prediction_prepare_frames = prediction_prepare_frames*frames_bool
label_loss = 0.1*label_loss_fn.calculate_loss(prediction_prepare_frames, labels_prepare_frames) + \
0.1*label_loss_fn.calculate_loss(prediction_prepare_video, labels_batch)
else:
label_loss = label_loss*0.0
elif "prediction_minmax" in result.keys():
predictions_minmax = result["prediction_minmax"]
predictions_min = tf.reduce_min(predictions_minmax,axis=1)
predictions_max = tf.reduce_max(predictions_minmax,axis=1)
epsilon = 10e-6
float_labels = tf.cast(labels_batch, tf.float32)
cross_entropy_loss = float_labels * tf.log(predictions_min + epsilon) + (
1 - float_labels) * tf.log(1 - predictions_max + epsilon)
frame_loss = tf.reduce_mean(tf.reduce_sum(tf.negative(cross_entropy_loss), 1))
label_loss = label_loss*0.0
else:
frame_loss = tf.constant(0.0)
tf.summary.scalar("label_loss", label_loss)
if "regularization_loss" in result.keys():
reg_loss = result["regularization_loss"]
else:
reg_loss = tf.constant(0.0)
reg_losses = tf.losses.get_regularization_losses()
if reg_losses:
reg_loss += tf.add_n(reg_losses)
if regularization_penalty != 0:
tf.summary.scalar("reg_loss", reg_loss)
# Adds update_ops (e.g., moving average updates in batch normalization) as
# a dependency to the train_op.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
if "update_ops" in result.keys():
update_ops += result["update_ops"]
if update_ops:
with tf.control_dependencies(update_ops):
barrier = tf.no_op(name="gradient_barrier")
with tf.control_dependencies([barrier]):
label_loss = tf.identity(label_loss)
# Incorporate the L2 weight penalties etc.
final_loss = regularization_penalty * reg_loss + label_loss + frame_loss + append_loss
if FLAGS.gradient=="my":
opt = tf.train.AdamOptimizer(learning_rate=learning_rate)
variables_to_train = tf_variables.trainable_variables()
top_grads, top_vars = mygradients(final_loss, variables_to_train, global_step=global_step, name="mygradients_net")
grads_and_vars = list(zip(top_grads, top_vars))
train_op = opt.apply_gradients(grads_and_vars, global_step=global_step)
else:
optimizer = optimizer_class(learning_rate)
train_op = slim.learning.create_train_op(
final_loss,
optimizer,
global_step=global_step,
clip_gradient_norm=clip_gradient_norm)
tf.add_to_collection("global_step", global_step)
tf.add_to_collection("loss", final_loss)
tf.add_to_collection("reg_loss", reg_loss)
tf.add_to_collection("bottleneck", bottle_neck)
tf.add_to_collection("predictions", predictions)
tf.add_to_collection("input_batch_raw", model_input_raw)
tf.add_to_collection("input_batch", model_input)
tf.add_to_collection("num_frames", num_frames)
tf.add_to_collection("labels", tf.cast(labels_batch, tf.float32))
tf.add_to_collection("train_op", train_op)
class Trainer(object):
"""A Trainer to train a Tensorflow graph."""
def __init__(self, cluster, task, train_dir, log_device_placement=True):
""""Creates a Trainer.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
self.cluster = cluster
self.task = task
self.is_master = (task.type == "master" and task.index == 0)
self.train_dir = train_dir
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=FLAGS.gpu)
self.config = tf.ConfigProto(log_device_placement=log_device_placement)
if self.is_master and self.task.index > 0:
raise StandardError("%s: Only one replica of master expected",
task_as_string(self.task))
def run(self, start_new_model=False):
"""Performs training on the currently defined Tensorflow graph.
Returns:
A tuple of the training Hit@1 and the training PERR.
"""
if self.is_master and start_new_model:
self.remove_training_directory(self.train_dir)
target, device_fn = self.start_server_if_distributed()
meta_filename = self.get_meta_filename(start_new_model, self.train_dir)
with tf.Graph().as_default() as graph:
if meta_filename:
saver = self.recover_model(meta_filename)
with tf.device(device_fn):
if not meta_filename:
saver = self.build_model()
global_step = tf.get_collection("global_step")[0]
loss = tf.get_collection("loss")[0]
reg_loss = tf.get_collection("reg_loss")[0]
predictions = tf.get_collection("predictions")[0]
labels = tf.get_collection("labels")[0]
train_op = tf.get_collection("train_op")[0]
init_op = tf.global_variables_initializer()
sv = tf.train.Supervisor(
graph,
logdir=self.train_dir,
init_op=init_op,
is_chief=self.is_master,
global_step=global_step,
save_model_secs=15 * 60,
save_summaries_secs=120,
saver=saver)
logging.info("%s: Starting managed session.", task_as_string(self.task))
with sv.managed_session(target, config=self.config) as sess:
try:
logging.info("%s: Entering training loop.", task_as_string(self.task))
while not sv.should_stop():
batch_start_time = time.time()
_, global_step_val, loss_val, reg_loss_val, predictions_val, labels_val = sess.run(
[train_op, global_step, loss, reg_loss, predictions, labels])
seconds_per_batch = time.time() - batch_start_time
if self.is_master:
examples_per_second = labels_val.shape[0] / seconds_per_batch
hit_at_one = eval_util.calculate_hit_at_one(predictions_val,
labels_val)
perr = eval_util.calculate_precision_at_equal_recall_rate(
predictions_val, labels_val)
gap = eval_util.calculate_gap(predictions_val, labels_val)
logging.info(
"%s: training step " + str(global_step_val) + "| Hit@1: " +
("%.2f" % hit_at_one) + " PERR: " + ("%.2f" % perr) + " GAP: " +
("%.2f" % gap) + " Loss: " + str(loss_val) +
" RegLoss: " + str(reg_loss_val),
task_as_string(self.task))
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_Hit@1", hit_at_one),
global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_Perr", perr), global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("model/Training_GAP", gap), global_step_val)
sv.summary_writer.add_summary(
utils.MakeSummary("global_step/Examples/Second",
examples_per_second), global_step_val)
sv.summary_writer.flush()
except tf.errors.OutOfRangeError:
logging.info("%s: Done training -- epoch limit reached.",
task_as_string(self.task))
logging.info("%s: Exited training loop.", task_as_string(self.task))
sv.Stop()
def start_server_if_distributed(self):
"""Starts a server if the execution is distributed."""
if self.cluster:
logging.info("%s: Starting trainer within cluster %s.",
task_as_string(self.task), self.cluster.as_dict())
server = start_server(self.cluster, self.task)
target = server.target
device_fn = tf.train.replica_device_setter(
ps_device="/job:ps",
worker_device="/job:%s/task:%d" % (self.task.type, self.task.index),
cluster=self.cluster)
else:
target = ""
device_fn = ""
return (target, device_fn)
def remove_training_directory(self, train_dir):
"""Removes the training directory."""
try:
logging.info(
"%s: Removing existing train directory.",
task_as_string(self.task))
gfile.DeleteRecursively(train_dir)
except:
logging.error(
"%s: Failed to delete directory " + train_dir +
" when starting a new model. Please delete it manually and" +
" try again.", task_as_string(self.task))
def get_meta_filename(self, start_new_model, train_dir):
if start_new_model:
logging.info("%s: Flag 'start_new_model' is set. Building a new model.",
task_as_string(self.task))
return None
latest_checkpoint = tf.train.latest_checkpoint(train_dir)
if not latest_checkpoint:
logging.info("%s: No checkpoint file found. Building a new model.",
task_as_string(self.task))
return None
meta_filename = latest_checkpoint + ".meta"
if not gfile.Exists(meta_filename):
logging.info("%s: No meta graph file found. Building a new model.",
task_as_string(self.task))
return None
else:
return meta_filename
def recover_model(self, meta_filename):
logging.info("%s: Restoring from meta graph file %s",
task_as_string(self.task), meta_filename)
return tf.train.import_meta_graph(meta_filename)
def build_model(self):
"""Find the model and build the graph."""
# Convert feature_names and feature_sizes to lists of values.
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
distill_names, distill_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.distill_names, FLAGS.distill_sizes)
if FLAGS.frame_features:
reader1 = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader1 = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
reader2 = readers.YT8MAggregatedFeatureReader(
feature_names=distill_names, feature_sizes=distill_sizes)
# Find the model.
model = find_class_by_name(FLAGS.model,
[frame_level_models, video_level_models])()
label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])()
optimizer_class = find_class_by_name(FLAGS.optimizer, [tf.train])
build_graph(reader1=reader1,
reader2=reader2,
model=model,
optimizer_class=optimizer_class,
clip_gradient_norm=FLAGS.clip_gradient_norm,
train_data_pattern=FLAGS.train_data_pattern,
distill_data_pattern=FLAGS.distill_data_pattern,
label_loss_fn=label_loss_fn,
base_learning_rate=FLAGS.base_learning_rate,
learning_rate_decay=FLAGS.learning_rate_decay,
learning_rate_decay_examples=FLAGS.learning_rate_decay_examples,
regularization_penalty=FLAGS.regularization_penalty,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size,
num_epochs=FLAGS.num_epochs)
logging.info("%s: Built graph.", task_as_string(self.task))
return tf.train.Saver(max_to_keep=2, keep_checkpoint_every_n_hours=1)
class ParameterServer(object):
"""A parameter server to serve variables in a distributed execution."""
def __init__(self, cluster, task):
"""Creates a ParameterServer.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
self.cluster = cluster
self.task = task
def run(self):
"""Starts the parameter server."""
logging.info("%s: Starting parameter server within cluster %s.",
task_as_string(self.task), self.cluster.as_dict())
server = start_server(self.cluster, self.task)
server.join()
def start_server(cluster, task):
"""Creates a Server.
Args:
cluster: A tf.train.ClusterSpec if the execution is distributed.
None otherwise.
task: A TaskSpec describing the job type and the task index.
"""
if not task.type:
raise ValueError("%s: The task type must be specified." %
task_as_string(task))
if task.index is None:
raise ValueError("%s: The task index must be specified." %
task_as_string(task))
# Create and start a server.
return tf.train.Server(
tf.train.ClusterSpec(cluster),
protocol="grpc",
job_name=task.type,
task_index=task.index)
def task_as_string(task):
return "/job:%s/task:%s" % (task.type, task.index)
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object,), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.",
task_as_string(task), tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
Trainer(cluster, task, FLAGS.train_dir, FLAGS.log_device_placement).run(
start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
if __name__ == "__main__":
app.run()
================================================
FILE: youtube-8m-zhangteng/train_scripts/run-attention-pooling-lstm-a.sh
================================================
CUDA_VISIBLE_DEVICES=1 python train.py \
--train_data_pattern='/Youtube-8M/data/frame/train/*.tfrecord' \
--train_dir='../model/frame_level_lstm_glu2_model' \
--frame_features=True \
--feature_names="rgb, audio" \
--feature_sizes="1024, 128" \
--model='LstmGlu2Model' \
--video_level_classifier_model=MoeModel \
--moe_num_mixtures=8 \
--batch_size=128 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-zhangteng/train_scripts/run-attention-pooling-lstm-s.sh
================================================
CUDA_VISIBLE_DEVICES=1 python train.py \
--train_data_pattern='/Youtube-8M/data/frame/train/*.tfrecord' \
--train_dir='../model/frame_level_lstm_multiscale2_model' \
--frame_features=True \
--feature_names="rgb, audio" \
--feature_sizes="1024, 128" \
--model='LstmMultiscale2Model' \
--video_level_classifier_model=MoeModel \
--norm=False \
--moe_num_extend=4 \
--moe_num_mixtures=4 \
--batch_size=128 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-zhangteng/train_scripts/run-attention-pooling-lstm2lstm.sh
================================================
CUDA_VISIBLE_DEVICES=1 python train.py \
--train_data_pattern='/Youtube-8M/data/frame/train/*.tfrecord' \
--train_dir='../model/frame_level_input_extend_model' \
--frame_features=True \
--feature_names="rgb, audio" \
--feature_sizes="1024, 128" \
--model='InputExtendModel' \
--video_level_classifier_model=MoeExtendModel \
--moe_num_extend=4 \
--moe_num_mixtures=8 \
--batch_size=128 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-zhangteng/train_scripts/run-attention-pooling.sh
================================================
CUDA_VISIBLE_DEVICES=1 python train.py \
--train_data_pattern='/Youtube-8M/data/frame/train/*.tfrecord' \
--train_dir='../model/frame_level_lstm_extend8_model' \
--frame_features=True \
--feature_names="rgb, audio" \
--feature_sizes="1024, 128" \
--model='LstmExtendModel' \
--video_level_classifier_model=MoeExtendModel \
--moe_num_extend=8 \
--moe_num_mixtures=8 \
--batch_size=128 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-zhangteng/train_scripts/run-bilstm-a.sh
================================================
CUDA_VISIBLE_DEVICES=1 python train.py \
--train_data_pattern='/Youtube-8M/data/frame/train/*.tfrecord' \
--train_dir='../model/frame_level_lstm_biglu_model' \
--frame_features=True \
--feature_names="rgb, audio" \
--feature_sizes="1024, 128" \
--model='LstmBigluModel' \
--moe_num_mixtures=4 \
--batch_size=128 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-zhangteng/train_scripts/run-cascade-76-lstm-a.sh
================================================
CUDA_VISIBLE_DEVICES=1 python train.py \
--train_data_pattern='/Youtube-8M/distillation_v2/frame/train/*.tfrecord' \
--train_dir='../model/frame_level_lstm_glu2_distillchain_v2_model' \
--frame_features=True \
--feature_names="rgb, audio" \
--feature_sizes="1024, 128" \
--model='LstmGlu2Model' \
--video_level_classifier_model=MoeDistillChainModel \
--moe_num_mixtures=8 \
--batch_size=128 \
--base_learning_rate=0.001 \
--distillation_features=True \
--distillation_type=0
================================================
FILE: youtube-8m-zhangteng/train_scripts/run-cascade-76-lstm-s.sh
================================================
CUDA_VISIBLE_DEVICES=1 python train.py \
--train_data_pattern='/Youtube-8M/distillation_v2/frame/train/*.tfrecord' \
--train_dir='../model/frame_level_lstm_gate_distillchain_v2_model' \
--frame_features=True \
--feature_names="rgb, audio" \
--feature_sizes="1024, 128" \
--model='LstmGateModel' \
--video_level_classifier_model=MoeDistillChainModel \
--moe_num_mixtures=8 \
--batch_size=128 \
--base_learning_rate=0.001 \
--distillation_features=True \
--distillation_type=0
================================================
FILE: youtube-8m-zhangteng/train_scripts/run-cascade-76-lstm.sh
================================================
CUDA_VISIBLE_DEVICES=1 python train.py \
--train_data_pattern='/Youtube-8M/distillation_v2/frame/train/*.tfrecord' \
--train_dir='../model/frame_level_lstm_distillchain_v2_model' \
--frame_features=True \
--feature_names="rgb, audio" \
--feature_sizes="1024, 128" \
--model='LstmModel' \
--video_level_classifier_model=MoeDistillChainModel \
--moe_num_mixtures=8 \
--lstm_layers=1 \
--batch_size=128 \
--base_learning_rate=0.001 \
--distillation_features=True \
--distillation_type=0
================================================
FILE: youtube-8m-zhangteng/train_scripts/run-cascade-attention-pooling.sh
================================================
CUDA_VISIBLE_DEVICES=1 python train.py \
--train_data_pattern='/Youtube-8M/distillation/frame/train/*.tfrecord' \
--train_dir='../model/frame_level_lstm_extend_distillchain_model' \
--frame_features=True \
--feature_names="rgb, audio" \
--feature_sizes="1024, 128" \
--model='LstmExtendModel' \
--video_level_classifier_model=MoeExtendDistillChainModel \
--moe_num_extend=8
--moe_num_mixtures=8 \
--batch_size=128 \
--base_learning_rate=0.001 \
--distillation_features=True \
--distillation_type=0
================================================
FILE: youtube-8m-zhangteng/train_scripts/run-cascade-chaining-cnn-layer2.sh
================================================
CUDA_VISIBLE_DEVICES=1 python train.py \
--train_data_pattern='/Youtube-8M/distillation/frame/train/*.tfrecord' \
--train_dir='../model/frame_level_cnndcc_distillchain_model' \
--frame_features=True \
--feature_names="rgb, audio" \
--feature_sizes="1024, 128" \
--model='CnnDCCDistillChainModel' \
--moe_layers=2
--moe_num_mixtures=4 \
--batch_size=128 \
--base_learning_rate=0.001 \
--distillation_features=True \
--distillation_type=0
================================================
FILE: youtube-8m-zhangteng/train_scripts/run-cascade-chaining-video-normalize.sh
================================================
CUDA_VISIBLE_DEVICES=1 python train.py \
--train_data_pattern='/Youtube-8M/distillation/video/train/*.tfrecord' \
--train_dir='../model/video_level_distillchainnorm2_model' \
--frame_features=False \
--feature_names="mean_rgb, mean_audio" \
--feature_sizes="1024, 128" \
--model='MoeDistillChainNorm2Model' \
--moe_num_mixtures=8 \
--batch_size=128 \
--base_learning_rate=0.001 \
--distillation_features=True \
--distillation_type=0
================================================
FILE: youtube-8m-zhangteng/train_scripts/run-cascade-lstm-s-split.sh
================================================
CUDA_VISIBLE_DEVICES=1 python train.py \
--train_data_pattern='/Youtube-8M/distillation/frame/train/*.tfrecord' \
--train_dir='../model/frame_level_lstm_gate_distillsplit_model' \
--frame_features=True \
--feature_names="rgb, audio" \
--feature_sizes="1024, 128" \
--model='LstmGateModel' \
--video_level_classifier_model=MoeDistillSplitModel \
--moe_num_mixtures=8 \
--batch_size=128 \
--base_learning_rate=0.001 \
--distillation_features=True \
--distillation_type=0
================================================
FILE: youtube-8m-zhangteng/train_scripts/run-cascade-lstm-s.sh
================================================
CUDA_VISIBLE_DEVICES=1 python train.py \
--train_data_pattern='/Youtube-8M/distillation/frame/train/*.tfrecord' \
--train_dir='../model/frame_level_lstm_gate_distillchain_model' \
--frame_features=True \
--feature_names="rgb, audio" \
--feature_sizes="1024, 128" \
--model='LstmGateModel' \
--video_level_classifier_model=MoeDistillChainModel \
--moe_num_mixtures=8 \
--batch_size=128 \
--base_learning_rate=0.001 \
--distillation_features=True \
--distillation_type=0
================================================
FILE: youtube-8m-zhangteng/train_scripts/run-cascade-lstm.sh
================================================
CUDA_VISIBLE_DEVICES=1 python train.py \
--train_data_pattern='/Youtube-8M/distillation/frame/train/*.tfrecord' \
--train_dir='../model/frame_level_lstm_distillchain_model' \
--frame_features=True \
--feature_names="rgb, audio" \
--feature_sizes="1024, 128" \
--model='LstmModel' \
--video_level_classifier_model=MoeDistillChainModel \
--moe_num_mixtures=8 \
--lstm_layers=1 \
--batch_size=128 \
--base_learning_rate=0.001 \
--distillation_features=True \
--distillation_type=0
================================================
FILE: youtube-8m-zhangteng/train_scripts/run-cascade-multiscale-cnn-lstm-laery4.sh
================================================
CUDA_VISIBLE_DEVICES=1 python train.py \
--train_data_pattern='/Youtube-8M/distillation/frame/train/*.tfrecord' \
--train_dir='../model/frame_level_lstm_multiscale_distillchain4_model' \
--frame_features=True \
--feature_names="rgb, audio" \
--feature_sizes="1024, 128" \
--model='LstmMultiscaleDitillChainModel' \
--moe_num_extend=4 \
--moe_num_mixtures=4 \
--cnn_cells=196 \
--batch_size=128 \
--base_learning_rate=0.001 \
--distillation_features=True \
--distillation_type=0
================================================
FILE: youtube-8m-zhangteng/train_scripts/run-cascade-multiscale-cnn-lstm-layer2.sh
================================================
CUDA_VISIBLE_DEVICES=1 python train.py \
--train_data_pattern='/Youtube-8M/distillation/frame/train/*.tfrecord' \
--train_dir='../model/frame_level_lstm_multiscale_distillchain_model' \
--frame_features=True \
--feature_names="rgb, audio" \
--feature_sizes="1024, 128" \
--model='LstmMultiscaleDitillChainModel' \
--moe_num_extend=2 \
--moe_num_mixtures=4 \
--cnn_cells=256 \
--batch_size=128 \
--base_learning_rate=0.001 \
--distillation_features=True \
--distillation_type=0
================================================
FILE: youtube-8m-zhangteng/train_scripts/run-chaining-video-add-confident.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_data_pattern='/Youtube-8M/data/video/train/*.tfrecord' \
--train_dir='../model/video_level_moemix4_relabel_model' \
--model='MoeMix4Model' \
--feature_names="mean_rgb, mean_audio" \
--feature_sizes="1024, 128" \
--moe_num_mixtures=4 \
--moe_layers=3 \
--class_size=100 \
--loss_function="loss_relabel"
================================================
FILE: youtube-8m-zhangteng/train_scripts/run-chaining-video-infrequent-softmax.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_data_pattern='/Youtube-8M/data/video/train/*.tfrecord' \
--train_dir='../model/video_level_moesoftmax_model' \
--model='MoeSoftmaxModel' \
--feature_names="mean_rgb, mean_audio" \
--feature_sizes="1024, 128" \
--moe_num_mixtures=4 \
--moe_layers=3 \
--class_size=100
================================================
FILE: youtube-8m-zhangteng/train_scripts/run-chaining-video-normal.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_data_pattern='/Youtube-8M/data/video/train/*.tfrecord' \
--train_dir='../model/video_level_moemix4_model' \
--model='MoeMix4Model' \
--feature_names="mean_rgb, mean_audio" \
--feature_sizes="1024, 128" \
--moe_num_mixtures=4 \
--moe_layers=3 \
--class_size=100
================================================
FILE: youtube-8m-zhangteng/train_scripts/run-chaining-video-vertical.sh
================================================
CUDA_VISIBLE_DEVICES=0 python train.py \
--train_data_pattern='/Youtube-8M/data/video/train/*.tfrecord' \
--train_dir='../model/video_level_moeknowledge_model' \
--model='MoeKnowledgeModel' \
--feature_names="mean_rgb, mean_audio" \
--feature_sizes="1024, 128" \
--moe_num_mixtures=4 \
--moe_layers=3 \
--class_size=100
================================================
FILE: youtube-8m-zhangteng/train_scripts/run-lstm-random-augmentation.sh
================================================
CUDA_VISIBLE_DEVICES=1 python train.py \
--train_data_pattern='/Youtube-8M/data/frame/train/*.tfrecord' \
--train_dir='../model/frame_level_lstm_random_model' \
--frame_features=True \
--feature_names="rgb, audio" \
--feature_sizes="1024, 128" \
--model='LstmRandomModel' \
--video_level_classifier_model=MoeModel \
--moe_num_mixtures=8 \
--batch_size=128 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-zhangteng/train_scripts/run-lstm-s.sh
================================================
CUDA_VISIBLE_DEVICES=1 python train.py \
--train_data_pattern='/Youtube-8M/data/frame/train/*.tfrecord' \
--train_dir='../model/frame_level_lstm_gate_model' \
--frame_features=True \
--feature_names="rgb, audio" \
--feature_sizes="1024, 128" \
--model='LstmGateModel' \
--video_level_classifier_model=MoeModel \
--moe_num_mixtures=8 \
--batch_size=128 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-zhangteng/train_scripts/run-multiscale-cnn-lstm.sh
================================================
CUDA_VISIBLE_DEVICES=1 python train.py \
--train_data_pattern='/Youtube-8M/data/frame/train/*.tfrecord' \
--train_dir='../model/frame_level_lstm_multiscale_model' \
--frame_features=True \
--feature_names="rgb, audio" \
--feature_sizes="1024, 128" \
--model='LstmMultiscaleModel' \
--video_level_classifier_model=MoeModel \
--moe_num_extend=4 \
--moe_num_mixtures=4 \
--batch_size=128 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-zhangteng/train_scripts/run-temporal-segment-lstm.sh
================================================
CUDA_VISIBLE_DEVICES=1 python train.py \
--train_data_pattern='/Youtube-8M/data/frame/train/*.tfrecord' \
--train_dir='../model/frame_level_lstm_layer_model' \
--frame_features=True \
--feature_names="rgb, audio" \
--feature_sizes="1024, 128" \
--model='LstmLayerModel' \
--video_level_classifier_model=MoeModel \
--lstm_length=10 \
--moe_num_mixtures=8 \
--batch_size=128 \
--base_learning_rate=0.001
================================================
FILE: youtube-8m-zhangteng/training_utils/del.py
================================================
import os
check = {}
check_list = []
for filename in os.listdir("."):
if filename.endswith("meta"):
checkpoint = int(filename.split("-")[1].split(".")[0])
check_list.append(checkpoint)
check_list.sort()
for checkpoint in check_list:
if check.has_key(checkpoint / 4000):
print checkpoint
else:
check[checkpoint / 4000] = True
================================================
FILE: youtube-8m-zhangteng/training_utils/select.py
================================================
import os
import sys
every = int(sys.argv[1])
check = {}
check_list = []
for filename in os.listdir("."):
if filename.endswith("meta"):
checkpoint = int(filename.split("-")[1].split(".")[0])
check_list.append(checkpoint)
check_list.sort()
for checkpoint in check_list:
if check.has_key(checkpoint / every):
pass
else:
check[checkpoint / every] = True
print checkpoint
================================================
FILE: youtube-8m-zhangteng/utils.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains a collection of util functions for training and evaluating.
"""
import numpy
import tensorflow as tf
from tensorflow import logging
def Dequantize(feat_vector, max_quantized_value=2, min_quantized_value=-2):
"""Dequantize the feature from the byte format to the float format.
Args:
feat_vector: the input 1-d vector.
max_quantized_value: the maximum of the quantized value.
min_quantized_value: the minimum of the quantized value.
Returns:
A float vector which has the same shape as feat_vector.
"""
assert max_quantized_value > min_quantized_value
quantized_range = max_quantized_value - min_quantized_value
scalar = quantized_range / 255.0
bias = (quantized_range / 512.0) + min_quantized_value
return feat_vector * scalar + bias
def MakeSummary(name, value):
"""Creates a tf.Summary proto with the given name and value."""
summary = tf.Summary()
val = summary.value.add()
val.tag = str(name)
val.simple_value = float(value)
return summary
def AddGlobalStepSummary(summary_writer,
global_step_val,
global_step_info_dict,
summary_scope="Eval"):
"""Add the global_step summary to the Tensorboard.
Args:
summary_writer: Tensorflow summary_writer.
global_step_val: a int value of the global step.
global_step_info_dict: a dictionary of the evaluation metrics calculated for
a mini-batch.
summary_scope: Train or Eval.
Returns:
A string of this global_step summary
"""
this_hit_at_one = global_step_info_dict["hit_at_one"]
this_perr = global_step_info_dict["perr"]
this_loss = global_step_info_dict["loss"]
examples_per_second = global_step_info_dict.get("examples_per_second", -1)
summary_writer.add_summary(
MakeSummary("GlobalStep/" + summary_scope + "_Hit@1", this_hit_at_one),
global_step_val)
summary_writer.add_summary(
MakeSummary("GlobalStep/" + summary_scope + "_Perr", this_perr),
global_step_val)
summary_writer.add_summary(
MakeSummary("GlobalStep/" + summary_scope + "_Loss", this_loss),
global_step_val)
if examples_per_second != -1:
summary_writer.add_summary(
MakeSummary("GlobalStep/" + summary_scope + "_Example_Second",
examples_per_second), global_step_val)
summary_writer.flush()
info = ("global_step {0} | Batch Hit@1: {1:.3f} | Batch PERR: {2:.3f} | Batch Loss: {3:.3f} "
"| Examples_per_sec: {4:.3f}").format(
global_step_val, this_hit_at_one, this_perr, this_loss,
examples_per_second)
return info
def AddEpochSummary(summary_writer,
global_step_val,
epoch_info_dict,
summary_scope="Eval"):
"""Add the epoch summary to the Tensorboard.
Args:
summary_writer: Tensorflow summary_writer.
global_step_val: a int value of the global step.
epoch_info_dict: a dictionary of the evaluation metrics calculated for the
whole epoch.
summary_scope: Train or Eval.
Returns:
A string of this global_step summary
"""
epoch_id = epoch_info_dict["epoch_id"]
avg_hit_at_one = epoch_info_dict["avg_hit_at_one"]
avg_perr = epoch_info_dict["avg_perr"]
avg_loss = epoch_info_dict["avg_loss"]
aps = epoch_info_dict["aps"]
gap = epoch_info_dict["gap"]
mean_ap = numpy.mean(aps)
summary_writer.add_summary(
MakeSummary("Epoch/" + summary_scope + "_Avg_Hit@1", avg_hit_at_one),
global_step_val)
summary_writer.add_summary(
MakeSummary("Epoch/" + summary_scope + "_Avg_Perr", avg_perr),
global_step_val)
summary_writer.add_summary(
MakeSummary("Epoch/" + summary_scope + "_Avg_Loss", avg_loss),
global_step_val)
summary_writer.add_summary(
MakeSummary("Epoch/" + summary_scope + "_MAP", mean_ap),
global_step_val)
summary_writer.add_summary(
MakeSummary("Epoch/" + summary_scope + "_GAP", gap),
global_step_val)
summary_writer.flush()
info = ("epoch/eval number {0} | Avg_Hit@1: {1:.3f} | Avg_PERR: {2:.3f} "
"| MAP: {3:.3f} | GAP: {4:.3f} | Avg_Loss: {5:3f}").format(
epoch_id, avg_hit_at_one, avg_perr, mean_ap, gap, avg_loss)
return info
def GetListOfFeatureNamesAndSizes(feature_names, feature_sizes):
"""Extract the list of feature names and the dimensionality of each feature
from string of comma separated values.
Args:
feature_names: string containing comma separated list of feature names
feature_sizes: string containing comma separated list of feature sizes
Returns:
List of the feature names and list of the dimensionality of each feature.
Elements in the first/second list are strings/integers.
"""
list_of_feature_names = [
feature_names.strip() for feature_names in feature_names.split(',')]
list_of_feature_sizes = [
int(feature_sizes) for feature_sizes in feature_sizes.split(',')]
if len(list_of_feature_names) != len(list_of_feature_sizes):
logging.error("length of the feature names (=" +
str(len(list_of_feature_names)) + ") != length of feature "
"sizes (=" + str(len(list_of_feature_sizes)) + ")")
return list_of_feature_names, list_of_feature_sizes
================================================
FILE: youtube-8m-zhangteng/video_level_models.py
================================================
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains model definitions."""
import math
import models
import tensorflow as tf
import numpy as np
import utils
from tensorflow import flags
import tensorflow.contrib.slim as slim
FLAGS = flags.FLAGS
flags.DEFINE_integer(
"moe_num_mixtures", 8,
"The number of mixtures (excluding the dummy 'expert') used for MoeModel.")
flags.DEFINE_integer(
"moe_num_extend", 8,
"The number of attention outputs, used for MoeExtendModel.")
flags.DEFINE_string("moe_method", "none",
"The pooling method used in the DBoF cluster layer. "
"used for MoeMaxModel.")
flags.DEFINE_integer(
"class_size", 200,
"The dimention of prediction projection, used for all chain models.")
flags.DEFINE_integer(
"encoder_size", 100,
"The dimention of prediction encoder, used for all mix models.")
flags.DEFINE_integer(
"hidden_size_1", 100,
"The size of the first hidden layer, used forAutoEncoderModel.")
flags.DEFINE_integer(
"hidden_channels", 3,
"The number of hidden layers, only used in early experiment.")
flags.DEFINE_integer(
"moe_layers", 1,
"The number of combine layers, used for combine related models.")
flags.DEFINE_integer(
"softmax_bound", 1000,
"The number of labels to be a group, only used for MoeSoftmaxModel and MoeDistillSplitModel.")
flags.DEFINE_bool(
"moe_group", False,
"Whether to split the 4716 labels into different groups, used in MoeMix4Model and MoeNoiseModel")
flags.DEFINE_float("noise_std", 0.2, "the standard deviation of noise added to the input.")
flags.DEFINE_float("ensemble_w", 1.0, "ensemble weight used in distill chain models.")
class LogisticModel(models.BaseModel):
"""Logistic model with L2 regularization."""
def create_model(self, model_input, vocab_size, l2_penalty=1e-8, **unused_params):
"""Creates a logistic model.
Args:
model_input: 'batch' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes."""
output = slim.fully_connected(
model_input, vocab_size, activation_fn=tf.nn.sigmoid,
weights_regularizer=slim.l2_regularizer(l2_penalty))
return {"predictions": output}
class MoeModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self,
model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
shape = model_input.get_shape().as_list()
if FLAGS.frame_features:
model_input = tf.reshape(model_input,[-1,shape[-1]])
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates")
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts")
"""
gate_w = tf.get_variable("gate_w", [shape[1], vocab_size * (num_mixtures + 1)], tf.float32,
initializer=tf.contrib.layers.xavier_initializer())
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(gate_w))
gate_activations = tf.matmul(model_input,gate_w)
expert_w = tf.get_variable("expert_w", [shape[1], vocab_size * num_mixtures], tf.float32,
initializer=tf.contrib.layers.xavier_initializer())
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(expert_w))
expert_v = tf.get_variable("expert_v", [vocab_size * num_mixtures], tf.float32,
initializer=tf.constant_initializer(0.0))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(expert_v))
expert_activations = tf.nn.xw_plus_b(model_input,expert_w,expert_v)"""
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
probabilities_by_class_and_batch = tf.reshape(probabilities_by_class_and_batch,
[-1, vocab_size])
final_probabilities = tf.reshape(probabilities_by_class_and_batch,
[-1, vocab_size])
return {"predictions": final_probabilities}
class MoeDistillModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self,
model_input,
vocab_size,
distill_labels=None,
num_mixtures=None,
l2_penalty=1e-8,
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
shape = model_input.get_shape().as_list()
if FLAGS.frame_features:
model_input = tf.reshape(model_input,[-1,shape[-1]])
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates")
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts")
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
probabilities_by_class_and_batch = tf.reshape(probabilities_by_class_and_batch,
[-1, vocab_size])
final_sub_probabilities = tf.reshape(probabilities_by_class_and_batch,
[-1, vocab_size])
if distill_labels is not None:
expert_gate = slim.fully_connected(
model_input,
vocab_size,
activation_fn=tf.nn.sigmoid,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="expert_gate")
expert_gate = expert_gate*0.8 + 0.1
final_probabilities = distill_labels*(1.0-expert_gate) + final_sub_probabilities*expert_gate
tf.summary.histogram("expert_gate/activations", expert_gate)
else:
final_probabilities = final_sub_probabilities
return {"predictions": final_probabilities, "predictions_class": final_sub_probabilities}
class MoeDistillEmbeddingModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self,
model_input,
vocab_size,
distill_labels=None,
num_mixtures=None,
l2_penalty=1e-8,
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
"""
embedding_mat = np.loadtxt("./resources/embedding_matrix.model.gz")
embedding_mat = tf.cast(embedding_mat,dtype=tf.float32)
bound = FLAGS.softmax_bound
vocab_size_1 = bound
probabilities_by_distill = distill_labels[:, :vocab_size_1]
embedding_mat = embedding_mat[:vocab_size_1, :]
labels_smooth = tf.matmul(probabilities_by_distill, embedding_mat)
probabilities_by_smooth_1 = (labels_smooth[:, :vocab_size_1] - probabilities_by_distill)/tf.reduce_sum(probabilities_by_distill,axis=1,keep_dims=True)
probabilities_by_smooth_2 = labels_smooth[:, vocab_size_1:]/tf.reduce_sum(probabilities_by_distill,axis=1,keep_dims=True)
labels_smooth = tf.concat((probabilities_by_smooth_1, probabilities_by_smooth_2), axis=1)"""
expert_gate = slim.fully_connected(
distill_labels,
1,
activation_fn=tf.nn.sigmoid,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="expert_gate")
#final_probabilities = tf.clip_by_value(distill_labels + labels_smooth, 0.0, 1.0)
final_probabilities = distill_labels
return {"predictions": final_probabilities}
class MoeDistillChainModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self,
model_input,
vocab_size,
distill_labels=None,
num_mixtures=None,
l2_penalty=1e-8,
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
class_size = 256
shape = model_input.get_shape().as_list()
if distill_labels is not None:
class_input = slim.fully_connected(
distill_labels,
class_size,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="class_inputs")
model_input = tf.concat((model_input,class_input),axis=1)
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates")
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts")
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
probabilities_by_class_and_batch = tf.reshape(probabilities_by_class_and_batch,
[-1, vocab_size])
final_probabilities = tf.reshape(probabilities_by_class_and_batch,
[-1, vocab_size])
final_probabilities = final_probabilities*FLAGS.ensemble_w + distill_labels*(1.0-FLAGS.ensemble_w)
return {"predictions": final_probabilities}
class MoeDistillChainNormModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self,
model_input,
vocab_size,
distill_labels=None,
num_mixtures=None,
l2_penalty=1e-8,
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
class_size = 256
model_input = tf.nn.l2_normalize(model_input,dim=1)
if distill_labels is not None:
class_input = slim.fully_connected(
distill_labels,
class_size,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="class_inputs")
class_input = class_input/tf.reduce_sum(distill_labels,axis=1,keep_dims=True)
class_input = tf.nn.l2_normalize(class_input,dim=1)
model_input = tf.concat((model_input,class_input),axis=1)
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates")
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts")
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
probabilities_by_class_and_batch = tf.reshape(probabilities_by_class_and_batch,
[-1, vocab_size])
final_probabilities = tf.reshape(probabilities_by_class_and_batch,
[-1, vocab_size])
final_probabilities = final_probabilities*FLAGS.ensemble_w + distill_labels*(1.0-FLAGS.ensemble_w)
return {"predictions": final_probabilities}
class MoeDistillChainNorm2Model(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self,
model_input,
vocab_size,
distill_labels=None,
num_mixtures=None,
l2_penalty=1e-8,
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
class_size = 256
model_input = tf.nn.l2_normalize(model_input,dim=1)
if distill_labels is not None:
class_input = slim.fully_connected(
distill_labels,
class_size,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="class_inputs")
class_input = class_input/tf.reduce_sum(distill_labels,axis=1,keep_dims=True)
class_input = tf.nn.l2_normalize(class_input,dim=1)
model_input = tf.concat((model_input,class_input),axis=1)
gate_activations = slim.fully_connected(
model_input,
vocab_size * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates")
expert_activations = slim.fully_connected(
model_input,
vocab_size * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts")
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
probabilities_by_class_and_batch = tf.reshape(probabilities_by_class_and_batch,
[-1, vocab_size])
final_probabilities = tf.reshape(probabilities_by_class_and_batch,
[-1, vocab_size])
final_probabilities = final_probabilities*FLAGS.ensemble_w + distill_labels*(1.0-FLAGS.ensemble_w)
return {"predictions": final_probabilities}
class MoeDistillSplitModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self,
model_input,
vocab_size,
distill_labels=None,
num_mixtures=None,
l2_penalty=1e-8,
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
bound = FLAGS.softmax_bound
vocab_size_1 = bound
class_size = 256
probabilities_by_distill = distill_labels[:,vocab_size_1:]
class_input = slim.fully_connected(
probabilities_by_distill,
class_size,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="class_inputs")
#class_input = tf.nn.l2_normalize(class_input, dim=1)
model_input = tf.concat((model_input,class_input),axis=1)
gate_activations = slim.fully_connected(
model_input,
vocab_size_1 * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates")
expert_activations = slim.fully_connected(
model_input,
vocab_size_1 * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts")
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
probabilities_by_class_and_batch = tf.reshape(probabilities_by_class_and_batch,
[-1, vocab_size_1])
final_probabilities = tf.concat((probabilities_by_class_and_batch, probabilities_by_distill), axis=1)
final_probabilities = final_probabilities*FLAGS.ensemble_w + distill_labels*(1.0-FLAGS.ensemble_w)
return {"predictions": final_probabilities}
class MoeDistillSplit2Model(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self,
model_input,
vocab_size,
distill_labels=None,
num_mixtures=None,
l2_penalty=1e-8,
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
bound = FLAGS.softmax_bound
vocab_size_1 = bound
class_size = 256
probabilities_by_distill = distill_labels[:,vocab_size_1:]
probabilities_by_residual = tf.clip_by_value(1.0-tf.reduce_sum(probabilities_by_distill,axis=1,keep_dims=True), 0.0, 1.0)
probabilities_by_distill_residual = tf.concat((probabilities_by_residual,probabilities_by_distill), axis=1)
class_input = slim.fully_connected(
probabilities_by_distill_residual,
class_size,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="class_inputs")
class_input = tf.nn.l2_normalize(class_input, dim=1)
model_input = tf.concat((model_input,class_input),axis=1)
gate_activations = slim.fully_connected(
model_input,
vocab_size_1 * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates")
expert_activations = slim.fully_connected(
model_input,
vocab_size_1 * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts")
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
probabilities_by_class_and_batch = tf.reshape(probabilities_by_class_and_batch,
[-1, vocab_size_1])
final_probabilities = tf.concat((probabilities_by_class_and_batch, probabilities_by_distill), axis=1)
final_probabilities = final_probabilities*FLAGS.ensemble_w + distill_labels*(1.0-FLAGS.ensemble_w)
return {"predictions": final_probabilities}
class MoeDistillSplit3Model(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self,
model_input,
vocab_size,
distill_labels=None,
num_mixtures=None,
l2_penalty=1e-8,
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
bound = FLAGS.softmax_bound
vocab_size_1 = bound
vocab_size_2 = vocab_size - vocab_size_1
class_size = 256
probabilities_by_distill = distill_labels[:,:vocab_size_1]
probabilities_by_residual = distill_labels[:,vocab_size_1:]
feature_size = model_input.get_shape().as_list()[1]
model_input = slim.fully_connected(
model_input,
feature_size,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="model_inputs")
model_input = tf.nn.l2_normalize(model_input, dim=1)
gate_activations_1 = slim.fully_connected(
model_input,
vocab_size_1 * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-1")
expert_activations_1 = slim.fully_connected(
model_input,
vocab_size_1 * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-1")
gating_distribution_1 = tf.nn.softmax(tf.reshape(
gate_activations_1,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution_1 = tf.nn.sigmoid(tf.reshape(
expert_activations_1,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
probabilities_by_class_and_batch_1 = tf.reduce_sum(
gating_distribution_1[:, :num_mixtures] * expert_distribution_1, 1)
probabilities_by_class_and_batch_1 = tf.reshape(probabilities_by_class_and_batch_1,
[-1, vocab_size_1])
probabilities_by_class = tf.concat((probabilities_by_class_and_batch_1, probabilities_by_residual), axis=1)
class_input = slim.fully_connected(
probabilities_by_distill,
class_size,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="class_inputs")
class_input = tf.nn.l2_normalize(class_input, dim=1)
model_input = tf.concat((model_input,class_input),axis=1)
gate_activations = slim.fully_connected(
model_input,
vocab_size_2 * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates")
expert_activations = slim.fully_connected(
model_input,
vocab_size_2 * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts")
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
probabilities_by_class_and_batch = tf.reshape(probabilities_by_class_and_batch,
[-1, vocab_size_2])
final_probabilities = tf.concat((probabilities_by_distill, probabilities_by_class_and_batch), axis=1)
final_probabilities = final_probabilities*FLAGS.ensemble_w + distill_labels*(1.0-FLAGS.ensemble_w)
return {"predictions": final_probabilities, "predictions_class": probabilities_by_class}
class MoeDistillSplit4Model(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def create_model(self,
model_input,
vocab_size,
distill_labels=None,
num_mixtures=None,
l2_penalty=1e-8,
**unused_params):
"""Creates a Mixture of (Logistic) Experts model.
The model consists of a per-class softmax distribution over a
configurable number of logistic classifiers. One of the classifiers in the
mixture is not trained, and always predicts 0.
Args:
model_input: 'batch_size' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
num_mixtures: The number of mixtures (excluding a dummy 'expert' that
always predicts the non-existence of an entity).
l2_penalty: How much to penalize the squared magnitudes of parameter
values.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes.
"""
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
bound = FLAGS.softmax_bound
vocab_size_1 = bound
vocab_size_2 = vocab_size - vocab_size_1
class_size = 256
probabilities_by_distill = distill_labels[:,:vocab_size_1]
probabilities_by_residual = distill_labels[:,vocab_size_1:]
gate_activations_1 = slim.fully_connected(
model_input,
vocab_size_1 * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates-1")
expert_activations_1 = slim.fully_connected(
model_input,
vocab_size_1 * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts-1")
gating_distribution_1 = tf.nn.softmax(tf.reshape(
gate_activations_1,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution_1 = tf.nn.sigmoid(tf.reshape(
expert_activations_1,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
probabilities_by_class_and_batch_1 = tf.reduce_sum(
gating_distribution_1[:, :num_mixtures] * expert_distribution_1, 1)
probabilities_by_class_and_batch_1 = tf.reshape(probabilities_by_class_and_batch_1,
[-1, vocab_size_1])
probabilities_by_class = tf.concat((probabilities_by_class_and_batch_1, probabilities_by_residual), axis=1)
class_input = slim.fully_connected(
probabilities_by_distill,
class_size,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="class_inputs")
model_input = tf.concat((model_input,class_input),axis=1)
gate_activations = slim.fully_connected(
model_input,
vocab_size_2 * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates")
expert_activations = slim.fully_connected(
model_input,
vocab_size_2 * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts")
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
probabilities_by_class_and_batch = tf.reshape(probabilities_by_class_and_batch,
[-1, vocab_size_2])
final_probabilities = tf.concat((probabilities_by_distill, probabilities_by_class_and_batch), axis=1)
final_probabilities = final_probabilities*FLAGS.ensemble_w + distill_labels*(1.0-FLAGS.ensemble_w)
return {"predictions": final_probabilities, "predictions_class": probabilities_by_class}
class MoeSoftmaxModel(models.BaseModel):
"""A softmax over a mixture of logistic models (with L2 regularization)."""
def sub_model(self,
model_input,
vocab_size,
num_mixtures=None,
l2_penalty=1e-8,
name="",
**unused_params):
num_mixtures = num_mixtures or FLAGS.moe_num_mixtures
class_size = FLAGS.class_size
bound = FLAGS.softmax_bound
vocab_size_1 = bound
gate_activations = slim.fully_connected(
model_input,
vocab_size_1 * (num_mixtures + 1),
activation_fn=None,
biases_initializer=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="gates"+name)
expert_activations = slim.fully_connected(
model_input,
vocab_size_1 * num_mixtures,
activation_fn=None,
weights_regularizer=slim.l2_regularizer(l2_penalty),
scope="experts"+name)
gating_distribution = tf.nn.softmax(tf.reshape(
gate_activations,
[-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
expert_distribution = tf.nn.sigmoid(tf.reshape(
expert_activations,
[-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
probabilities_by_class_and_batch = tf.reduce_sum(
gating_distribution[:, :num_mixtures] * expert_distribution, 1)
probabilities_by_sigmoid = tf.reshape(probabilities_by_class_and_batch,
[-1, vocab_size_1])
vocab_size_2 = vocab_size - bound
class_size = vocab_size_2
channels = 1
probabilities_by_softmax = []
for i in range(channels):
if i 0, "self.feature_names is empty!"
assert len(self.feature_names) == len(self.feature_sizes), \
"length of feature_names (={}) != length of feature_sizes (={})".format( \
len(self.feature_names), len(self.feature_sizes))
feature_map = {"video_id": tf.FixedLenFeature([], tf.string),
"labels": tf.VarLenFeature(tf.int64)}
for feature_index in range(num_features):
feature_map[self.feature_names[feature_index]] = tf.FixedLenFeature(
[self.feature_sizes[feature_index]], tf.float32)
features = tf.parse_example(serialized_examples, features=feature_map)
labels = tf.sparse_to_indicator(features["labels"], self.num_classes)
labels.set_shape([None, self.num_classes])
concatenated_features = tf.concat([
features[feature_name] for feature_name in self.feature_names], 1)
return features["video_id"], concatenated_features, labels, tf.ones([tf.shape(serialized_examples)[0]])