[ { "path": "ReadMe.md", "content": "# 竞赛列表\n+ [2017 知乎看山杯机器学习挑战赛](https://www.biendata.com/competition/zhihu/)\n" }, { "path": "zhihu-text-classification-master/data_process/.idea/.name", "content": "data_process" }, { "path": "zhihu-text-classification-master/data_process/.idea/data_process.iml", "content": "\n\n \n \n \n \n \n \n \n" }, { "path": "zhihu-text-classification-master/data_process/.idea/deployment.xml", "content": "\n\n \n \n \n \n \n \n \n \n \n \n \n" }, { "path": "zhihu-text-classification-master/data_process/.idea/encodings.xml", "content": "\n\n \n \n \n" }, { "path": "zhihu-text-classification-master/data_process/.idea/misc.xml", "content": "\n\n \n \n \n \n \n \n \n \n \n \n \n" }, { "path": "zhihu-text-classification-master/data_process/.idea/modules.xml", "content": "\n\n \n \n \n \n \n" }, { "path": "zhihu-text-classification-master/data_process/.idea/workspace.xml", "content": "\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n true\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n 1504013687331\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n" }, { "path": "zhihu-text-classification-master/data_process/README.md", "content": "## 数据处理\n\n1.把比赛提供的所有数据解压到 raw_data/ 目录下。
\n2.按照顺序依次执行各个 .py，不带任何参数。
\n 或者在当前目录下输入下面命令运行所有文件：
\n dos2unix run_all_data_process.sh # 使用cygwin工具dos2unix将script改为unix格式
\n sh run_all_data_process.sh
\n3.环境依赖(下面是我使用的版本)
\n- numpy\t\t1.12.1\n- pandas \t0.19.2\n- word2vec\t0.9.1\n- tqdm\t\t4.11.2\n\n\n### embed2ndarray.py\n赛方提供了txt格式的词向量和字向量，这里把embedding矩阵转成 np.ndarray 形式，分别保存为 data/word_embedding.npy 和 data/char_embedding.npy。在赛方提供的词向量基础上，添加 '\\' 和 '\\' 两个特殊符号。其中 '\\' 用于将序列补全到固定长度， '\\' 用于替换低频词（字）。\n用 pd.Series 保存词(字)对应 embedding 中的行号(id),存储在 data/sr_word2id.pkl 和 data/sr_char2id.pkl 中。\n\n### question_and_topic_2id.py\n把问题和话题转为id形式，保存在 data/sr_question2id.pkl 和 data/sr_id2question.pkl 中。\n\n### char2id.py\n利用上面得到的 sr_char2id，把所有问题的字转为对应的id, 存储为\ndata/ch_train_title.npy\ndata/ch_train_content.npy\ndata/ch_eval_title.npy\ndata/ch_eval_content.npy\n\n### word2id.py\n同 char2id.py\n\n### creat_batch_data.py\n把所有的数据按照 batch_size(128) 进行打包，固定seed，随机取 10 万样本作为验证集。每个batch存储为一个 npz 文件，包括 X, y 两部分。\n这里所有的序列都进行了截断，长度不足的用0进行padding到固定长度。\n保存位置：\nwd_train_path = '../data/wd-data/data_train/'\nwd_valid_path = '../data/wd-data/data_valid/'\nwd_test_path = '../data/wd-data/data_test/'\nch_train_path = '../data/ch-data/data_train/'\nch_valid_path = '../data/ch-data/data_valid/'\nch_test_path = '../data/ch-data/data_test/'\n\n\n### creat_batch_seg.py\n和 creat_batch_data.py 相同，只是对 content 部分进行句子划分。用于分层模型。\n划分句子长度：\nwd_title_len = 30, wd_sent_len = 30, wd_doc_len = 10.(即content划分为10个句子，每个句子长度为30个词)\nch_title_len = 52, ch_sent_len = 52, ch_doc_len = 10.\n不划分句子：\nwd_title_len = 30, wd_content_len = 150.\nch_title_len = 52, ch_content_len = 300.\n\n\n### To do\n- 在数据读取中使用 tfrecord 文件进行数据读取。这样能够随时改变 batch_size， 而且 shuffle 会比使用 numpy 更加均匀。\n- 添加序列长度信息。在这里所有的序列都截断或者padding为固定长度，在误差计算中没有处理padding部分，可能会使准确率下降。在使用 dynamic_rnn 的时候加上 sequence_length 信息，在计算的时候忽略 padding 部分。同时结合 tf.train.SequenceExample() 和 tf.train.batch() 自动 padding，也可以减少数据量。" }, { "path": "zhihu-text-classification-master/data_process/char2id.py", "content": "# -*- coding:utf-8 -*-\n\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport numpy as np\nimport pandas as pd\nimport pickle\nfrom multiprocessing import Pool\nfrom tqdm import tqdm\nimport time\n\n\nsave_path = '../data/'\nwith open(save_path + 'sr_char2id.pkl', 'rb') as inp:\n sr_id2char = pickle.load(inp)\n sr_char2id = pickle.load(inp)\ndict_char2id = dict()\nfor i in range(len(sr_char2id)):\n dict_char2id[sr_char2id.index[i]] = sr_char2id.values[i]\n\n\ndef get_id(char):\n \"\"\"获取 char 所对应的 id.\n 如果该字不在字典中，用1进行替换。\n \"\"\"\n if char not in dict_char2id:\n return 1\n else:\n return dict_char2id[char]\n\n\ndef get_id4chars(chars):\n \"\"\"把 chars 转为 对应的 id\"\"\"\n chars = chars.strip().split(',') # 先分开字\n ids = list(map(get_id, chars)) # 获取id\n return ids\n\n\ndef test_char2id():\n \"\"\"把测试集的所有字转成对应的id。\"\"\"\n time0 = time.time()\n print('Processing eval data.')\n df_eval = pd.read_csv('../raw_data/question_eval_set.txt', sep='\\t', usecols=[0, 1, 3],\n names=['question_id', 'char_title', 'char_content'], dtype={'question_id': object})\n print('test question number %d' % len(df_eval))\n # 没有 title 的问题用 content 来替换\n na_title_indexs = list()\n for i in range(len(df_eval)):\n char_title = df_eval.char_title.values[i]\n if type(char_title) is float:\n na_title_indexs.append(i)\n print('There are %d test questions without title.' % len(na_title_indexs))\n for na_index in na_title_indexs:\n df_eval.at[na_index, 'char_title'] = df_eval.at[na_index, 'char_content']\n # 没有 content 的问题用 title 来替换\n na_content_indexs = list()\n for i in tqdm(range(len(df_eval))):\n char_content = df_eval.char_content.values[i]\n if type(char_content) is float:\n na_content_indexs.append(i)\n print('There are %d test questions without content.' % len(na_content_indexs))\n for na_index in tqdm(na_content_indexs):\n df_eval.at[na_index, 'char_content'] = df_eval.at[na_index, 'char_title']\n # 转为 id 形式\n p = Pool()\n eval_title = np.asarray(p.map(get_id4chars, df_eval.char_title.values))\n np.save('../data/ch_eval_title.npy', eval_title)\n eval_content = np.asarray(p.map(get_id4chars, df_eval.char_content.values))\n np.save('../data/ch_eval_content.npy', eval_content)\n p.close()\n p.join()\n print('Finished changing the eval chars to ids. Costed time %g s' % (time.time()-time0))\n\n\ndef train_char2id():\n \"\"\"把训练集的所有字转成对应的id。\"\"\"\n time0 = time.time()\n print('Processing train data.')\n df_train = pd.read_csv('../raw_data/question_train_set.txt', sep='\\t', usecols=[0, 1, 3],\n names=['question_id', 'char_title', 'char_content'], dtype={'question_id': object})\n print('training question number %d ' % len(df_train))\n # 没有 content 的问题用 title 来替换\n na_content_indexs = list()\n for i in tqdm(range(len(df_train))):\n char_content = df_train.char_content.values[i]\n if type(char_content) is float:\n na_content_indexs.append(i)\n print('There are %d train questions without content.' % len(na_content_indexs))\n for na_index in tqdm(na_content_indexs):\n df_train.at[na_index, 'char_content'] = df_train.at[na_index, 'char_title']\n # 没有 title 的问题， 与词一样丢弃下面样本\n na_title_indexs = [328877, 422123, 633584, 768738, 818616, 876828, 1273673, 1527297,\n 1636237, 1682969, 2052477, 2628516, 2657464, 2904162, 2993517]\n for i in range(len(df_train)):\n char_title = df_train.char_title.values[i]\n if type(char_title) is float:\n na_title_indexs.append(i)\n print('There are %d train questions without title.' % len(na_title_indexs))\n df_train = df_train.drop(na_title_indexs)\n print('After dropping, training question number(should be 2999952) = %d' % len(df_train))\n # 转为 id 形式\n p = Pool()\n train_title = np.asarray(list(p.map(get_id4chars, df_train.char_title.values)))\n np.save('../data/ch_train_title.npy', train_title)\n train_content = np.asarray(p.map(get_id4chars, df_train.char_content.values))\n np.save('../data/ch_train_content.npy', train_content)\n p.close()\n p.join()\n print('Finished changing the training chars to ids. Costed time %g s' % (time.time() - time0))\n\n\nif __name__ == '__main__':\n test_char2id()\n train_char2id()\n\n\n\n\n\n\n" }, { "path": "zhihu-text-classification-master/data_process/creat_batch_data.py", "content": "# -*- coding:utf-8 -*-\n\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport numpy as np\nimport pandas as pd\nimport pickle\nfrom multiprocessing import Pool\nimport sys\nimport os\n\nsys.path.append('../')\nfrom data_helpers import pad_X30\nfrom data_helpers import pad_X150\nfrom data_helpers import pad_X52\nfrom data_helpers import pad_X300\nfrom data_helpers import train_batch\nfrom data_helpers import eval_batch\n\n\"\"\" 把所有的数据按照 batch_size(128) 进行打包。取 10万 样本作为验证集。\nword_title_len = 30.\nword_content_len = 150.\nchar_title_len = 52.\nchar_content_len = 300.\n\"\"\"\n\n\nwd_train_path = '../data/wd-data/data_train/'\nwd_valid_path = '../data/wd-data/data_valid/'\nwd_test_path = '../data/wd-data/data_test/'\nch_train_path = '../data/ch-data/data_train/'\nch_valid_path = '../data/ch-data/data_valid/'\nch_test_path = '../data/ch-data/data_test/'\npaths = [wd_train_path, wd_valid_path, wd_test_path,\n ch_train_path, ch_valid_path, ch_test_path]\nfor each in paths:\n if not os.path.exists(each):\n os.makedirs(each)\n\nwith open('../data/sr_topic2id.pkl', 'rb') as inp:\n sr_topic2id = pickle.load(inp)\n\ndict_topic2id = dict()\nfor i in range(len(sr_topic2id)):\n dict_topic2id[sr_topic2id.index[i]] = sr_topic2id.values[i]\n\n\ndef topics2ids(topics):\n \"\"\"把 chars 转为 对应的 id\"\"\"\n topics = topics.split(',')\n ids = list(map(lambda topic: dict_topic2id[topic], topics)) # 获取id\n return ids\n\n\ndef get_lables():\n \"\"\"获取训练集所有样本的标签。注意之前在处理数据时丢弃了部分没有 title 的样本。\"\"\"\n df_question_topic = pd.read_csv('../raw_data/question_topic_train_set.txt', sep='\\t',\n names=['questions', 'topics'], dtype={'questions': object, 'topics': object})\n na_title_indexs = [328877, 422123, 633584, 768738, 818616, 876828, 1273673, 1527297,\n 1636237, 1682969, 2052477, 2628516, 2657464, 2904162, 2993517]\n df_question_topic = df_question_topic.drop(na_title_indexs)\n p = Pool()\n y = p.map(topics2ids, df_question_topic.topics.values)\n p.close()\n p.join()\n return np.asarray(y)\n\n\n# word 数据打包\ndef wd_train_get_batch(title_len=30, content_len=150, batch_size=128):\n print('loading word train_title and train_content.')\n train_title = np.load('../data/wd_train_title.npy')\n train_content = np.load('../data/wd_train_content.npy')\n p = Pool()\n X_title = np.asarray(p.map(pad_X30, train_title))\n X_content = np.asarray(p.map(pad_X150, train_content))\n p.close()\n p.join()\n X = np.hstack([X_title, X_content])\n print('getting labels, this should cost minutes, please wait.')\n y = get_lables()\n print('y.shape=', y.shape)\n np.save('../data/y_tr.npy', y)\n # 划分验证集\n sample_num = X.shape[0]\n np.random.seed(13)\n valid_num = 100000\n new_index = np.random.permutation(sample_num)\n X = X[new_index]\n y = y[new_index]\n X_valid = X[:valid_num]\n y_valid = y[:valid_num]\n X_train = X[valid_num:]\n y_train = y[valid_num:]\n print('X_train.shape=', X_train.shape, 'y_train.shape=', y_train.shape)\n print('X_valid.shape=', X_valid.shape, 'y_valid.shape=', y_valid.shape)\n print('creating batch data.')\n # 验证集打batch\n sample_num = len(X_valid)\n print('valid_sample_num=%d' % sample_num)\n train_batch(X_valid, y_valid, wd_valid_path, batch_size)\n # 训练集打batch\n sample_num = len(X_train)\n print('train_sample_num=%d' % sample_num)\n train_batch(X_train, y_train, wd_train_path, batch_size)\n\n\ndef wd_test_get_batch(title_len=30, content_len=150, batch_size=128):\n eval_title = np.load('../data/wd_eval_title.npy')\n eval_content = np.load('../data/wd_eval_content.npy')\n p = Pool()\n X_title = np.asarray(p.map(pad_X30, eval_title))\n X_content = np.asarray(p.map(pad_X150, eval_content))\n p.close()\n p.join()\n X = np.hstack([X_title, X_content])\n sample_num = len(X)\n print('eval_sample_num=%d' % sample_num)\n eval_batch(X, wd_test_path, batch_size)\n\n\n# char 数据打包\ndef ch_train_get_batch(title_len=52, content_len=300, batch_size=128):\n print('loading char train_title and train_content.')\n train_title = np.load('../data/ch_train_title.npy')\n train_content = np.load('../data/ch_train_content.npy')\n p = Pool()\n X_title = np.asarray(p.map(pad_X52, train_title))\n X_content = np.asarray(p.map(pad_X300, train_content))\n p.close()\n p.join()\n X = np.hstack([X_title, X_content])\n y = np.load('../data/y_tr.npy')\n # 划分验证集\n sample_num = X.shape[0]\n np.random.seed(13)\n valid_num = 100000\n new_index = np.random.permutation(sample_num)\n X = X[new_index]\n y = y[new_index]\n X_valid = X[:valid_num]\n y_valid = y[:valid_num]\n X_train = X[valid_num:]\n y_train = y[valid_num:]\n print('X_train.shape=', X_train.shape, 'y_train.shape=', y_train.shape)\n print('X_valid.shape=', X_valid.shape, 'y_valid.shape=', y_valid.shape)\n # 验证集打batch\n print('creating batch data.')\n sample_num = len(X_valid)\n print('valid_sample_num=%d' % sample_num)\n train_batch(X_valid, y_valid, ch_valid_path, batch_size)\n # 训练集打batch\n sample_num = len(X_train)\n print('train_sample_num=%d' % sample_num)\n train_batch(X_train, y_train, ch_train_path, batch_size)\n\n\ndef ch_test_get_batch(title_len=52, content_len=300, batch_size=128):\n eval_title = np.load('../data/ch_eval_title.npy')\n eval_content = np.load('../data/ch_eval_content.npy')\n p = Pool()\n X_title = np.asarray(p.map(pad_X52, eval_title))\n X_content = np.asarray(p.map(pad_X300, eval_content))\n p.close()\n p.join()\n X = np.hstack([X_title, X_content])\n sample_num = len(X)\n print('eval_sample_num=%d' % sample_num)\n eval_batch(X, ch_test_path, batch_size)\n\n\nif __name__ == '__main__':\n wd_train_get_batch()\n wd_test_get_batch()\n ch_train_get_batch()\n ch_test_get_batch()\n" }, { "path": "zhihu-text-classification-master/data_process/creat_batch_seg.py", "content": "# -*- coding:utf-8 -*-\n\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport numpy as np\nfrom multiprocessing import Pool\nimport sys\nimport os\n\nsys.path.append('../')\nfrom data_helpers import pad_X30\nfrom data_helpers import pad_X52\nfrom data_helpers import wd_pad_cut_docs\nfrom data_helpers import ch_pad_cut_docs\nfrom data_helpers import train_batch\nfrom data_helpers import eval_batch\n\n\nwd_train_path = '../data/wd-data/seg_train/'\nwd_valid_path = '../data/wd-data/seg_valid/'\nwd_test_path = '../data/wd-data/seg_test/'\nch_train_path = '../data/ch-data/seg_train/'\nch_valid_path = '../data/ch-data/seg_valid/'\nch_test_path = '../data/ch-data/seg_test/'\npaths = [wd_train_path, wd_valid_path, wd_test_path,\n ch_train_path, ch_valid_path, ch_test_path]\nfor each in paths:\n if not os.path.exists(each):\n os.makedirs(each)\n\n\n# word 数据打包\ndef wd_train_get_batch(title_len=30, batch_size=128):\n print('loading word train_title and train_content, this should cost minutes, please wait.')\n train_title = np.load('../data/wd_train_title.npy')\n train_content = np.load('../data/wd_train_content.npy')\n p = Pool(6)\n X_title = np.asarray(p.map(pad_X30, train_title))\n X_content = np.asarray(p.map(wd_pad_cut_docs, train_content))\n p.close()\n p.join()\n X_content.shape = [-1, 30*10]\n X = np.hstack([X_title, X_content])\n y = np.load('../data/y_tr.npy')\n # 划分验证集\n sample_num = X.shape[0]\n np.random.seed(13)\n valid_num = 100000\n new_index = np.random.permutation(sample_num)\n X = X[new_index]\n y = y[new_index]\n X_valid = X[:valid_num]\n y_valid = y[:valid_num]\n X_train = X[valid_num:]\n y_train = y[valid_num:]\n print('X_train.shape=', X_train.shape, 'y_train.shape=', y_train.shape)\n print('X_valid.shape=', X_valid.shape, 'y_valid.shape=', y_valid.shape)\n # 验证集打 batch\n print('creating batch data.')\n sample_num = len(X_valid)\n print('valid_sample_num=%d' % sample_num)\n train_batch(X_valid, y_valid, wd_valid_path, batch_size)\n # 训练集打 batch\n sample_num = len(X_train)\n print('train_sample_num=%d' % sample_num)\n train_batch(X_train, y_train, wd_train_path, batch_size)\n\n\ndef wd_test_get_batch(title_len=30, batch_size=128):\n print('loading word eval_title and eval_content.')\n eval_title = np.load('../data/wd_eval_title.npy')\n eval_content = np.load('../data/wd_eval_content.npy')\n p = Pool(6)\n X_title = np.asarray(p.map(pad_X30, eval_title))\n X_content = np.asarray(p.map(wd_pad_cut_docs, eval_content))\n p.close()\n p.join()\n X_content.shape = [-1, 30*10]\n X = np.hstack([X_title, X_content])\n sample_num = len(X)\n print('eval_sample_num=%d' % sample_num)\n eval_batch(X, wd_test_path, batch_size)\n\n\n# char 数据打包\ndef ch_train_get_batch(title_len=52, batch_size=128):\n print('loading char train_title and train_content, this should cost minutes, please wait.')\n train_title = np.load('../data/ch_train_title.npy')\n train_content = np.load('../data/ch_train_content.npy')\n p = Pool(8)\n X_title = np.asarray(p.map(pad_X52, train_title))\n X_content = np.asarray(p.map(ch_pad_cut_docs, train_content))\n p.close()\n p.join()\n X_content.shape = [-1, 52*10]\n X = np.hstack([X_title, X_content])\n y = np.load('../data/y_tr.npy')\n # 划分验证集\n sample_num = X.shape[0]\n np.random.seed(13)\n valid_num = 100000\n new_index = np.random.permutation(sample_num)\n X = X[new_index]\n y = y[new_index]\n X_valid = X[:valid_num]\n y_valid = y[:valid_num]\n X_train = X[valid_num:]\n y_train = y[valid_num:]\n print('X_train.shape=', X_train.shape, 'y_train.shape=', y_train.shape)\n print('X_valid.shape=', X_valid.shape, 'y_valid.shape=', y_valid.shape)\n # 验证集打batch\n print('creating batch data.')\n sample_num = len(X_valid)\n print('valid_sample_num=%d' % sample_num)\n train_batch(X_valid, y_valid, ch_valid_path, batch_size)\n # 训练集打batch\n sample_num = len(X_train)\n print('train_sample_num=%d' % sample_num)\n train_batch(X_train, y_train, ch_train_path, batch_size)\n\n\ndef ch_test_get_batch(title_len=52, batch_size=128):\n print('loading char eval_title and eval_content.')\n eval_title = np.load('../data/ch_eval_title.npy')\n eval_content = np.load('../data/ch_eval_content.npy')\n p = Pool()\n X_title = np.asarray(p.map(pad_X52, eval_title))\n X_content = np.asarray(p.map(ch_pad_cut_docs, eval_content))\n p.close()\n p.join()\n X_content.shape = [-1, 52*10]\n X = np.hstack([X_title, X_content])\n sample_num = len(X)\n print('eval_sample_num=%d' % sample_num)\n eval_batch(X, ch_test_path, batch_size)\n\n\nif __name__ == '__main__':\n wd_train_get_batch()\n wd_test_get_batch()\n ch_train_get_batch()\n ch_test_get_batch()\n" }, { "path": "zhihu-text-classification-master/data_process/embed2ndarray.py", "content": "# -*- coding:utf-8 -*- \n\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport numpy as np\nimport pandas as pd\nimport word2vec\nimport pickle\nimport os\n\nSPECIAL_SYMBOL = ['', ''] # add these special symbols to word(char) embeddings.\n\n\ndef get_word_embedding():\n \"\"\"提取词向量，并保存至 ../data/word_embedding.npy\"\"\"\n print('getting the word_embedding.npy')\n wv = word2vec.load('../raw_data/word_embedding.txt')\n word_embedding = wv.vectors\n words = wv.vocab\n n_special_sym = len(SPECIAL_SYMBOL)\n sr_id2word = pd.Series(words, index=range(n_special_sym, n_special_sym + len(words)))\n sr_word2id = pd.Series(range(n_special_sym, n_special_sym + len(words)), index=words)\n # 添加特殊符号：:0, :1\n embedding_size = 256\n vec_special_sym = np.random.randn(n_special_sym, embedding_size)\n for i in range(n_special_sym):\n sr_id2word[i] = SPECIAL_SYMBOL[i]\n sr_word2id[SPECIAL_SYMBOL[i]] = i\n word_embedding = np.vstack([vec_special_sym, word_embedding])\n # 保存词向量\n save_path = '../data/'\n if not os.path.exists(save_path):\n os.makedirs(save_path)\n np.save(save_path + 'word_embedding.npy', word_embedding)\n # 保存词与id的对应关系\n with open(save_path + 'sr_word2id.pkl', 'wb') as outp:\n pickle.dump(sr_id2word, outp)\n pickle.dump(sr_word2id, outp)\n print('Saving the word_embedding.npy to ../data/word_embedding.npy')\n\n\ndef get_char_embedding():\n \"\"\"提取字向量，并保存至 ../data/char_embedding.npy\"\"\"\n print('getting the char_embedding.npy')\n wv = word2vec.load('../raw_data/char_embedding.txt')\n char_embedding = wv.vectors\n chars = wv.vocab\n n_special_sym = len(SPECIAL_SYMBOL)\n sr_id2char = pd.Series(chars, index=range(n_special_sym, n_special_sym + len(chars)))\n sr_char2id = pd.Series(range(n_special_sym, n_special_sym + len(chars)), index=chars)\n\n # 添加特殊符号：:0, :1\n embedding_size = 256\n\n vec_special_sym = np.random.randn(n_special_sym, embedding_size)\n for i in range(n_special_sym):\n sr_id2char[i] = SPECIAL_SYMBOL[i]\n sr_char2id[SPECIAL_SYMBOL[i]] = i\n char_embedding = np.vstack([vec_special_sym, char_embedding])\n # 保存字向量\n save_path = '../data/'\n if not os.path.exists(save_path):\n os.makedirs(save_path)\n np.save(save_path + 'char_embedding.npy', char_embedding)\n # 保存字与id的对应关系\n with open(save_path + 'sr_char2id.pkl', 'wb') as outp:\n pickle.dump(sr_id2char, outp)\n pickle.dump(sr_char2id, outp)\n print('Saving the char_embedding.npy to ../data/char_embedding.npy')\n\n\nif __name__ == '__main__':\n get_word_embedding()\n get_char_embedding()\n" }, { "path": "zhihu-text-classification-master/data_process/question_and_topic_2id.py", "content": "# -*- coding:utf-8 -*- \n\nimport pandas as pd\nimport pickle\nfrom itertools import chain\n\n\ndef question_and_topic_2id():\n \"\"\"把question和topic转成id形式并保存至 ../data/目录下。\"\"\"\n print('Changing the quetion and topic to id and save in sr_question2.pkl and sr_topic2id.pkl in ../data/')\n df_question_topic = pd.read_csv('../raw_data/question_topic_train_set.txt', sep='\\t', names=['question', 'topics'],\n dtype={'question': object, 'topics': object})\n df_question_topic.topics = df_question_topic.topics.apply(lambda tps: tps.split(','))\n save_path = '../data/'\n print('questino number = %d ' % len(df_question_topic))\n # 问题 id 按照给出的问题顺序编号\n questions = df_question_topic.question.values\n sr_question2id = pd.Series(range(len(questions)), index=questions) \n sr_id2question = pd.Series(questions, index=range(len(questions)))\n # topic 按照数量从大到小进行编号\n topics = df_question_topic.topics.values\n topics = list(chain(*topics))\n sr_topics = pd.Series(topics)\n topics_count = sr_topics.value_counts()\n topics = topics_count.index\n sr_topic2id = pd.Series(range(len(topics)),index=topics)\n sr_id2topic = pd.Series(topics, index=range(len(topics))) \n\n with open(save_path + 'sr_question2id.pkl', 'wb') as outp:\n pickle.dump(sr_question2id, outp)\n pickle.dump(sr_id2question, outp)\n with open(save_path + 'sr_topic2id.pkl', 'wb') as outp:\n pickle.dump(sr_topic2id, outp)\n pickle.dump(sr_id2topic, outp)\n print('Finished changing.')\n\n\nif __name__ == '__main__':\n question_and_topic_2id()\n" }, { "path": "zhihu-text-classification-master/data_process/run_all_data_process.sh", "content": "#!/usr/bin/env bash\necho -e \"\\033[44;37;5m RUNNING embed2ndarray.py\\033[0m \";\npython embed2ndarray.py;\necho -e \"\\033[44;37;5m RUNNING question_and_topic_2id.py\\033[0m \";\npython question_and_topic_2id.py;\necho -e \"\\033[44;37;5m RUNNING char2id.py\\033[0m \";\npython char2id.py;\necho -e \"\\033[44;37;5m RUNNING word2id.py\\033[0m \";\npython word2id.py;\necho -e \"\\033[44;37;5m RUNNING creat_batch_data.py\\033[0m \";\npython creat_batch_data.py;\necho -e \"\\033[44;37;5m RUNNING creat_batch_seg.py\\033[0m \";\npython creat_batch_seg.py;" }, { "path": "zhihu-text-classification-master/data_process/test.py", "content": "# -*- coding:utf-8 -*-\n\n\nfrom multiprocessing import Pool\nimport numpy as np\n\ndef func(a, b):\n return a+b\n\np = Pool()\na = [1,2,3]\nb = [4,5,6]\npara = zip(a,b)\nresult = p.map(func, para)\np.close()\np.join()\nprint result" }, { "path": "zhihu-text-classification-master/data_process/word2id.py", "content": "# -*- coding:utf-8 -*-\n\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport numpy as np\nimport pandas as pd\nimport pickle\nfrom multiprocessing import Pool\nfrom tqdm import tqdm\nimport time\n\nsave_path = '../data/'\nwith open(save_path + 'sr_word2id.pkl', 'rb') as inp:\n sr_id2word = pickle.load(inp)\n sr_word2id = pickle.load(inp)\ndict_word2id = dict()\nfor i in range(len(sr_word2id)):\n dict_word2id[sr_word2id.index[i]] = sr_word2id.values[i]\n\n\ndef get_id(word):\n \"\"\"获取 word 所对应的 id.\n 如果该词不在词典中，用 （对应的 ID 为 1 ）进行替换。\n \"\"\"\n if word not in dict_word2id:\n return 1\n else:\n return dict_word2id[word]\n\n\ndef get_id4words(words):\n \"\"\"把 words 转为 对应的 id\"\"\"\n words = words.strip().split(',') # 先分开词\n ids = list(map(get_id, words)) # 获取id\n return ids\n\n\ndef test_word2id():\n \"\"\"把测试集的所有词转成对应的id。\"\"\"\n time0 = time.time()\n print('Processing eval data.')\n df_eval = pd.read_csv('../raw_data/question_eval_set.txt', sep='\\t', usecols=[0, 2, 4],\n names=['question_id', 'word_title', 'word_content'], dtype={'question_id': object})\n print('test question number %d' % len(df_eval))\n # 没有 title 的问题用 content 来替换\n na_title_indexs = list()\n for i in range(len(df_eval)):\n word_title = df_eval.word_title.values[i]\n if type(word_title) is float:\n na_title_indexs.append(i)\n print('There are %d test questions without title.' % len(na_title_indexs))\n for na_index in na_title_indexs:\n df_eval.at[na_index, 'word_title'] = df_eval.at[na_index, 'word_content']\n # 没有 content 的问题用 title 来替换\n na_content_indexs = list()\n for i in tqdm(range(len(df_eval))):\n word_content = df_eval.word_content.values[i]\n if type(word_content) is float:\n na_content_indexs.append(i)\n print('There are %d test questions without content.' % len(na_content_indexs))\n for na_index in tqdm(na_content_indexs):\n df_eval.at[na_index, 'word_content'] = df_eval.at[na_index, 'word_title']\n # 转为 id 形式\n p = Pool()\n eval_title = np.asarray(p.map(get_id4words, df_eval.word_title.values))\n np.save('../data/wd_eval_title.npy', eval_title)\n eval_content = np.asarray(p.map(get_id4words, df_eval.word_content.values))\n np.save('../data/wd_eval_content.npy', eval_content)\n p.close()\n p.join()\n print('Finished changing the eval words to ids. Costed time %g s' % (time.time() - time0))\n\n\ndef train_word2id():\n \"\"\"把训练集的所有词转成对应的id。\"\"\"\n time0 = time.time()\n print('Processing train data.')\n df_train = pd.read_csv('../raw_data/question_train_set.txt', sep='\\t', usecols=[0, 2, 4],\n names=['question_id', 'word_title', 'word_content'], dtype={'question_id': object})\n print('training question number %d ' % len(df_train))\n # 没有 content 的问题用 title 来替换\n na_content_indexs = list()\n for i in tqdm(range(len(df_train))):\n word_content = df_train.word_content.values[i]\n if type(word_content) is float:\n na_content_indexs.append(i)\n print('There are %d train questions without content.' % len(na_content_indexs))\n for na_index in tqdm(na_content_indexs):\n df_train.at[na_index, 'word_content'] = df_train.at[na_index, 'word_title']\n # 没有 title 的问题， 丢弃\n na_title_indexs = list()\n for i in range(len(df_train)):\n word_title = df_train.word_title.values[i]\n if type(word_title) is float:\n na_title_indexs.append(i)\n print('There are %d train questions without title.' % len(na_title_indexs))\n df_train = df_train.drop(na_title_indexs)\n print('After dropping, training question number(should be 2999952) = %d' % len(df_train))\n # 转为 id 形式\n p = Pool()\n train_title = np.asarray(p.map(get_id4words, df_train.word_title.values))\n np.save('../data/wd_train_title.npy', train_title)\n train_content = np.asarray(p.map(get_id4words, df_train.word_content.values))\n np.save('../data/wd_train_content.npy', train_content)\n p.close()\n p.join()\n print('Finished changing the training words to ids. Costed time %g s' % (time.time() - time0))\n\n\nif __name__ == '__main__':\n test_word2id()\n train_word2id()\n" }, { "path": "zhihu-text-classification-master/models/wd_1_1_cnn_concat/__init__.py", "content": "# -*- coding:utf-8 -*- \n\n" }, { "path": "zhihu-text-classification-master/models/wd_1_1_cnn_concat/network.py", "content": "# -*- coding:utf-8 -*-\n\nimport tensorflow as tf\n\n\"\"\"wd_1_1_cnn_concat\ntitle 部分使用 TextCNN；content 部分使用 TextCNN； 两部分输出直接 concat。\n\"\"\"\n\n\nclass Settings(object):\n def __init__(self):\n self.model_name = 'wd_1_1_cnn_concat'\n self.title_len = 30\n self.content_len = 150\n self.filter_sizes = [2, 3, 4, 5, 7]\n self.n_filter = 256\n self.fc_hidden_size = 1024\n self.n_class = 1999\n self.summary_path = '../../summary/' + self.model_name + '/'\n self.ckpt_path = '../../ckpt/' + self.model_name + '/'\n\n\nclass TextCNN(object):\n \"\"\"\n title: inputs->textcnn->output_title\n content: inputs->textcnn->output_content\n concat[output_title, output_content] -> fc+bn+relu -> sigmoid_entropy.\n \"\"\"\n\n def __init__(self, W_embedding, settings):\n self.model_name = settings.model_name\n self.title_len = settings.title_len\n self.content_len = settings.content_len\n self.filter_sizes = settings.filter_sizes\n self.n_filter = settings.n_filter\n self.n_filter_total = self.n_filter * len(self.filter_sizes)\n self.n_class = settings.n_class\n self.fc_hidden_size = settings.fc_hidden_size\n self._global_step = tf.Variable(0, trainable=False, name='Global_Step')\n self.update_emas = list()\n # placeholders\n self._tst = tf.placeholder(tf.bool)\n self._keep_prob = tf.placeholder(tf.float32, [])\n self._batch_size = tf.placeholder(tf.int32, [])\n\n with tf.name_scope('Inputs'):\n self._X1_inputs = tf.placeholder(tf.int64, [None, self.title_len], name='X1_inputs')\n self._X2_inputs = tf.placeholder(tf.int64, [None, self.content_len], name='X2_inputs')\n self._y_inputs = tf.placeholder(tf.float32, [None, self.n_class], name='y_input')\n\n with tf.variable_scope('embedding'):\n self.embedding = tf.get_variable(name='embedding', shape=W_embedding.shape,\n initializer=tf.constant_initializer(W_embedding), trainable=True)\n self.embedding_size = W_embedding.shape[1]\n\n with tf.variable_scope('cnn_text'):\n output_title = self.cnn_inference(self._X1_inputs, self.title_len)\n\n with tf.variable_scope('hcnn_content'):\n output_content = self.cnn_inference(self._X2_inputs, self.content_len)\n\n with tf.variable_scope('fc-bn-layer'):\n output = tf.concat([output_title, output_content], axis=1)\n W_fc = self.weight_variable([self.n_filter_total * 2, self.fc_hidden_size], name='Weight_fc')\n tf.summary.histogram('W_fc', W_fc)\n h_fc = tf.matmul(output, W_fc, name='h_fc')\n beta_fc = tf.Variable(tf.constant(0.1, tf.float32, shape=[self.fc_hidden_size], name=\"beta_fc\"))\n tf.summary.histogram('beta_fc', beta_fc)\n fc_bn, update_ema_fc = self.batchnorm(h_fc, beta_fc, convolutional=False)\n self.update_emas.append(update_ema_fc)\n self.fc_bn_relu = tf.nn.relu(fc_bn, name=\"relu\")\n fc_bn_drop = tf.nn.dropout(self.fc_bn_relu, self.keep_prob)\n\n with tf.variable_scope('out_layer'):\n W_out = self.weight_variable([self.fc_hidden_size, self.n_class], name='Weight_out')\n tf.summary.histogram('Weight_out', W_out)\n b_out = self.bias_variable([self.n_class], name='bias_out')\n tf.summary.histogram('bias_out', b_out)\n self._y_pred = tf.nn.xw_plus_b(fc_bn_drop, W_out, b_out, name='y_pred') # 每个类别的分数 scores\n\n with tf.name_scope('loss'):\n self._loss = tf.reduce_mean(\n tf.nn.sigmoid_cross_entropy_with_logits(logits=self._y_pred, labels=self._y_inputs))\n tf.summary.scalar('loss', self._loss)\n\n self.saver = tf.train.Saver(max_to_keep=2)\n\n @property\n def tst(self):\n return self._tst\n\n @property\n def keep_prob(self):\n return self._keep_prob\n\n @property\n def batch_size(self):\n return self._batch_size\n\n @property\n def global_step(self):\n return self._global_step\n\n @property\n def X1_inputs(self):\n return self._X1_inputs\n\n @property\n def X2_inputs(self):\n return self._X2_inputs\n\n @property\n def y_inputs(self):\n return self._y_inputs\n\n @property\n def y_pred(self):\n return self._y_pred\n\n @property\n def loss(self):\n return self._loss\n\n def weight_variable(self, shape, name):\n \"\"\"Create a weight variable with appropriate initialization.\"\"\"\n initial = tf.truncated_normal(shape, stddev=0.1)\n return tf.Variable(initial, name=name)\n\n def bias_variable(self, shape, name):\n \"\"\"Create a bias variable with appropriate initialization.\"\"\"\n initial = tf.constant(0.1, shape=shape)\n return tf.Variable(initial, name=name)\n\n def batchnorm(self, Ylogits, offset, convolutional=False):\n \"\"\"batchnormalization.\n Args:\n Ylogits: 1D向量或者是3D的卷积结果。\n num_updates: 迭代的global_step\n offset：表示beta，全局均值；在 RELU 激活中一般初始化为 0.1。\n scale：表示lambda，全局方差；在 sigmoid 激活中需要，这 RELU 激活中作用不大。\n m: 表示batch均值；v:表示batch方差。\n bnepsilon：一个很小的浮点数，防止除以 0.\n Returns:\n Ybn: 和 Ylogits 的维度一样，就是经过 Batch Normalization 处理的结果。\n update_moving_everages：更新mean和variance，主要是给最后的 test 使用。\n \"\"\"\n exp_moving_avg = tf.train.ExponentialMovingAverage(0.999,\n self._global_step) # adding the iteration prevents from averaging across non-existing iterations\n bnepsilon = 1e-5\n if convolutional:\n mean, variance = tf.nn.moments(Ylogits, [0, 1, 2])\n else:\n mean, variance = tf.nn.moments(Ylogits, [0])\n update_moving_everages = exp_moving_avg.apply([mean, variance])\n m = tf.cond(self.tst, lambda: exp_moving_avg.average(mean), lambda: mean)\n v = tf.cond(self.tst, lambda: exp_moving_avg.average(variance), lambda: variance)\n Ybn = tf.nn.batch_normalization(Ylogits, m, v, offset, None, bnepsilon)\n return Ybn, update_moving_everages\n\n def cnn_inference(self, X_inputs, n_step):\n \"\"\"TextCNN 模型。\n Args:\n X_inputs: tensor.shape=(batch_size, n_step)\n Returns:\n title_outputs: tensor.shape=(batch_size, self.n_filter_total)\n \"\"\"\n inputs = tf.nn.embedding_lookup(self.embedding, X_inputs)\n inputs = tf.expand_dims(inputs, -1)\n pooled_outputs = list()\n for i, filter_size in enumerate(self.filter_sizes):\n with tf.variable_scope(\"conv-maxpool-%s\" % filter_size):\n # Convolution Layer\n filter_shape = [filter_size, self.embedding_size, 1, self.n_filter]\n W_filter = self.weight_variable(shape=filter_shape, name='W_filter')\n beta = self.bias_variable(shape=[self.n_filter], name='beta_filter')\n tf.summary.histogram('beta', beta)\n conv = tf.nn.conv2d(inputs, W_filter, strides=[1, 1, 1, 1], padding=\"VALID\", name=\"conv\")\n conv_bn, update_ema = self.batchnorm(conv, beta, convolutional=True) # 在激活层前面加 BN\n # Apply nonlinearity, batch norm scaling is not useful with relus\n # batch norm offsets are used instead of biases,使用 BN 层的 offset，不要 biases\n h = tf.nn.relu(conv_bn, name=\"relu\")\n # Maxpooling over the outputs\n pooled = tf.nn.max_pool(h, ksize=[1, n_step - filter_size + 1, 1, 1],\n strides=[1, 1, 1, 1], padding='VALID', name=\"pool\")\n pooled_outputs.append(pooled)\n self.update_emas.append(update_ema)\n h_pool = tf.concat(pooled_outputs, 3)\n h_pool_flat = tf.reshape(h_pool, [-1, self.n_filter_total])\n return h_pool_flat # shape = [batch_size, self.n_filter_total]\n\n\n# test the model\n# def test():\n# import numpy as np\n# print('Begin testing...')\n# settings = Settings()\n# W_embedding = np.random.randn(50, 10)\n# config = tf.ConfigProto()\n# config.gpu_options.allow_growth = True\n# batch_size = 128\n# with tf.Session(config=config) as sess:\n# model = TextCNN(W_embedding, settings)\n# optimizer = tf.train.AdamOptimizer(0.001)\n# train_op = optimizer.minimize(model.loss)\n# update_op = tf.group(*model.update_emas)\n# sess.run(tf.global_variables_initializer())\n# fetch = [model.loss, model.y_pred, train_op, update_op]\n# loss_list = list()\n# for i in xrange(100):\n# X1_batch = np.zeros((batch_size, 30), dtype=float)\n# X2_batch = np.zeros((batch_size, 150), dtype=float)\n# y_batch = np.zeros((batch_size, 1999), dtype=int)\n# _batch_size = len(y_batch)\n# feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n# model.batch_size: _batch_size, model.tst: False, model.keep_prob: 0.5}\n# loss, y_pred, _, _ = sess.run(fetch, feed_dict=feed_dict)\n# loss_list.append(loss)\n# print(i, loss)\n#\n# if __name__ == '__main__':\n# test()\n" }, { "path": "zhihu-text-classification-master/models/wd_1_1_cnn_concat/predict.py", "content": "# -*- coding:utf-8 -*-\n\nfrom __future__ import print_function\nfrom __future__ import division\nimport tensorflow as tf\nimport numpy as np\nfrom tqdm import tqdm\nimport os\nimport sys\nimport time\nimport network\n\nsys.path.append('../..')\nfrom evaluator import score_eval\n\nsettings = network.Settings()\ntitle_len = settings.title_len\nmodel_name = settings.model_name\nckpt_path = settings.ckpt_path\n\nlocal_scores_path = '../../local_scores/'\nscores_path = '../../scores/'\nif not os.path.exists(local_scores_path):\n os.makedirs(local_scores_path)\nif not os.path.exists(scores_path):\n os.makedirs(scores_path)\n\nembedding_path = '../../data/word_embedding.npy'\ndata_valid_path = '../../data/wd-data/data_valid/'\ndata_test_path = '../../data/wd-data/data_test/'\nva_batches = os.listdir(data_valid_path)\nte_batches = os.listdir(data_test_path) # batch 文件名列表\nn_va_batches = len(va_batches)\nn_te_batches = len(te_batches)\n\n\ndef get_batch(batch_id):\n \"\"\"get a batch from valid data\"\"\"\n new_batch = np.load(data_valid_path + str(batch_id) + '.npz')\n X_batch = new_batch['X']\n y_batch = new_batch['y']\n X1_batch = X_batch[:, :title_len]\n X2_batch = X_batch[:, title_len:]\n return [X1_batch, X2_batch, y_batch]\n\n\ndef get_test_batch(batch_id):\n \"\"\"get a batch from test data\"\"\"\n X_batch = np.load(data_test_path + str(batch_id) + '.npy')\n X1_batch = X_batch[:, :title_len]\n X2_batch = X_batch[:, title_len:]\n return [X1_batch, X2_batch]\n\n\ndef local_predict(sess, model):\n \"\"\"Test on the valid data.\"\"\"\n time0 = time.time()\n predict_labels_list = list() # 所有的预测结果\n marked_labels_list = list()\n predict_scores = list()\n for i in tqdm(xrange(n_va_batches)):\n [X1_batch, X2_batch, y_batch] = get_batch(i)\n marked_labels_list.extend(y_batch)\n _batch_size = len(X1_batch)\n fetches = [model.y_pred]\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n predict_labels = sess.run(fetches, feed_dict)[0]\n predict_scores.append(predict_labels)\n predict_labels = map(lambda label: label.argsort()[-1:-6:-1], predict_labels) # 取最大的5个下标\n predict_labels_list.extend(predict_labels)\n predict_label_and_marked_label_list = zip(predict_labels_list, marked_labels_list)\n precision, recall, f1 = score_eval(predict_label_and_marked_label_list)\n print('Local valid p=%g, r=%g, f1=%g' % (precision, recall, f1))\n predict_scores = np.vstack(np.asarray(predict_scores))\n local_scores_name = local_scores_path + model_name + '.npy'\n np.save(local_scores_name, predict_scores)\n print('local_scores.shape=', predict_scores.shape)\n print('Writed the scores into %s, time %g s' % (local_scores_name, time.time() - time0))\n\n\ndef predict(sess, model):\n \"\"\"Test on the test data.\"\"\"\n time0 = time.time()\n predict_scores = list()\n for i in tqdm(xrange(n_te_batches)):\n [X1_batch, X2_batch] = get_test_batch(i)\n _batch_size = len(X1_batch)\n fetches = [model.y_pred]\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n predict_labels = sess.run(fetches, feed_dict)[0]\n predict_scores.append(predict_labels)\n predict_scores = np.vstack(np.asarray(predict_scores))\n scores_name = scores_path + model_name + '.npy'\n np.save(scores_name, predict_scores)\n print('scores.shape=', predict_scores.shape)\n print('Writed the scores into %s, time %g s' % (scores_name, time.time() - time0))\n\n\ndef main(_):\n if not os.path.exists(ckpt_path + 'checkpoint'):\n print('there is not saved model, please check the ckpt path')\n exit()\n print('Loading model...')\n W_embedding = np.load(embedding_path)\n config = tf.ConfigProto()\n config.gpu_options.allow_growth = True\n with tf.Session(config=config) as sess:\n model = network.TextCNN(W_embedding, settings)\n model.saver.restore(sess, tf.train.latest_checkpoint(ckpt_path))\n print('Local predicting...')\n local_predict(sess, model)\n print('Test predicting...')\n predict(sess, model)\n\n\nif __name__ == '__main__':\n tf.app.run()\n" }, { "path": "zhihu-text-classification-master/models/wd_1_1_cnn_concat/train.py", "content": "# -*- coding:utf-8 -*-\n\nfrom __future__ import print_function\nfrom __future__ import division\nimport tensorflow as tf\nimport numpy as np\nfrom tqdm import tqdm\nimport os\nimport sys\nimport shutil\nimport time\nimport network\n\nsys.path.append('../..')\nfrom data_helpers import to_categorical\nfrom evaluator import score_eval\n\nflags = tf.flags\nflags.DEFINE_bool('is_retrain', False, 'if is_retrain is true, not rebuild the summary')\nflags.DEFINE_integer('max_epoch', 1, 'update the embedding after max_epoch, default: 1')\nflags.DEFINE_integer('max_max_epoch', 6, 'all training epoches, default: 6')\nflags.DEFINE_float('lr', 1e-3, 'initial learning rate, default: 1e-3')\nflags.DEFINE_float('decay_rate', 0.65, 'decay rate, default: 0.65')\nflags.DEFINE_float('keep_prob', 0.5, 'keep_prob for training, default: 0.5')\n# 正式\nflags.DEFINE_integer('decay_step', 15000, 'decay_step, default: 15000')\nflags.DEFINE_integer('valid_step', 10000, 'valid_step, default: 10000')\nflags.DEFINE_float('last_f1', 0.40, 'if valid_f1 > last_f1, save new model. default: 0.40')\n\n# 测试\n# flags.DEFINE_integer('decay_step', 1000, 'decay_step, default: 1000')\n# flags.DEFINE_integer('valid_step', 500, 'valid_step, default: 500')\n# flags.DEFINE_float('last_f1', 0.10, 'if valid_f1 > last_f1, save new model. default: 0.10')\nFLAGS = flags.FLAGS\n\nlr = FLAGS.lr\nlast_f1 = FLAGS.last_f1\nsettings = network.Settings()\ntitle_len = settings.title_len\nsummary_path = settings.summary_path\nckpt_path = settings.ckpt_path\nmodel_path = ckpt_path + 'model.ckpt'\n\nembedding_path = '../../data/word_embedding.npy'\ndata_train_path = '../../data/wd-data/data_train/'\ndata_valid_path = '../../data/wd-data/data_valid/'\ntr_batches = os.listdir(data_train_path) # batch 文件名列表\nva_batches = os.listdir(data_valid_path)\nn_tr_batches = len(tr_batches)\nn_va_batches = len(va_batches)\n\n# 测试\n# n_tr_batches = 1000\n# n_va_batches = 50\n\n\ndef get_batch(data_path, batch_id):\n \"\"\"get a batch from data_path\"\"\"\n new_batch = np.load(data_path + str(batch_id) + '.npz')\n X_batch = new_batch['X']\n y_batch = new_batch['y']\n X1_batch = X_batch[:, :title_len]\n X2_batch = X_batch[:, title_len:]\n return [X1_batch, X2_batch, y_batch]\n\n\ndef valid_epoch(data_path, sess, model):\n \"\"\"Test on the valid data.\"\"\"\n va_batches = os.listdir(data_path)\n n_va_batches = len(va_batches)\n _costs = 0.0\n predict_labels_list = list() # 所有的预测结果\n marked_labels_list = list()\n for i in range(n_va_batches):\n [X1_batch, X2_batch, y_batch] = get_batch(data_path, i)\n marked_labels_list.extend(y_batch)\n y_batch = to_categorical(y_batch)\n _batch_size = len(y_batch)\n fetches = [model.loss, model.y_pred]\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n _cost, predict_labels = sess.run(fetches, feed_dict)\n _costs += _cost\n predict_labels = list(map(lambda label: label.argsort()[-1:-6:-1], predict_labels)) # 取最大的5个下标\n predict_labels_list.extend(predict_labels)\n predict_label_and_marked_label_list = zip(predict_labels_list, marked_labels_list)\n precision, recall, f1 = score_eval(predict_label_and_marked_label_list)\n mean_cost = _costs / n_va_batches\n return mean_cost, precision, recall, f1\n\n\ndef train_epoch(data_train_path, sess, model, train_fetches, valid_fetches, train_writer, test_writer):\n global last_f1\n global lr\n time0 = time.time()\n batch_indexs = np.random.permutation(n_tr_batches) # shuffle the training data\n for batch in tqdm(range(n_tr_batches)):\n global_step = sess.run(model.global_step)\n if 0 == (global_step + 1) % FLAGS.valid_step:\n valid_cost, precision, recall, f1 = valid_epoch(data_valid_path, sess, model)\n print('Global_step=%d: valid cost=%g; p=%g, r=%g, f1=%g, time=%g s' % (\n global_step, valid_cost, precision, recall, f1, time.time() - time0))\n time0 = time.time()\n if f1 > last_f1:\n last_f1 = f1\n saving_path = model.saver.save(sess, model_path, global_step+1)\n print('saved new model to %s ' % saving_path)\n # training\n batch_id = batch_indexs[batch]\n [X1_batch, X2_batch, y_batch] = get_batch(data_train_path, batch_id)\n y_batch = to_categorical(y_batch)\n _batch_size = len(y_batch)\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: False, model.keep_prob: FLAGS.keep_prob}\n summary, _cost, _, _ = sess.run(train_fetches, feed_dict) # the cost is the mean cost of one batch\n # valid per 500 steps\n if 0 == (global_step + 1) % 500:\n train_writer.add_summary(summary, global_step)\n batch_id = np.random.randint(0, n_va_batches) # 随机选一个验证batch\n [X1_batch, X2_batch, y_batch] = get_batch(data_valid_path, batch_id)\n y_batch = to_categorical(y_batch)\n _batch_size = len(y_batch)\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n summary, _cost = sess.run(valid_fetches, feed_dict)\n test_writer.add_summary(summary, global_step)\n\n\ndef main(_):\n global ckpt_path\n global last_f1\n if not os.path.exists(ckpt_path):\n os.makedirs(ckpt_path)\n if not os.path.exists(summary_path):\n os.makedirs(summary_path)\n elif not FLAGS.is_retrain: # 重新训练本模型，删除以前的 summary\n shutil.rmtree(summary_path)\n os.makedirs(summary_path)\n if not os.path.exists(summary_path):\n os.makedirs(summary_path)\n\n print('1.Loading data...')\n W_embedding = np.load(embedding_path)\n print('training sample_num = %d' % n_tr_batches)\n print('valid sample_num = %d' % n_va_batches)\n\n # Initial or restore the model\n print('2.Building model...')\n config = tf.ConfigProto()\n config.gpu_options.allow_growth = True\n with tf.Session(config=config) as sess:\n model = network.TextCNN(W_embedding, settings)\n with tf.variable_scope('training_ops') as vs:\n learning_rate = tf.train.exponential_decay(FLAGS.lr, model.global_step, FLAGS.decay_step,\n FLAGS.decay_rate, staircase=True)\n # two optimizer: op1, update embedding; op2, do not update embedding.\n with tf.variable_scope('Optimizer1'):\n tvars1 = tf.trainable_variables()\n train_op1 = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(model.loss, global_step=model.global_step, var_list=tvars1)\n\n with tf.variable_scope('Optimizer2'):\n tvars2 = [tvar for tvar in tvars1 if 'embedding' not in tvar.name]\n train_op2 = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(model.loss, global_step=model.global_step, var_list=tvars2)\n\n update_op = tf.group(*model.update_emas)\n merged = tf.summary.merge_all() # summary\n train_writer = tf.summary.FileWriter(summary_path + 'train', sess.graph)\n test_writer = tf.summary.FileWriter(summary_path + 'test')\n training_ops = [v for v in tf.global_variables() if v.name.startswith(vs.name+'/')]\n\n # 如果已经保存过模型，导入上次的模型\n if os.path.exists(ckpt_path + \"checkpoint\"):\n print(\"Restoring Variables from Checkpoint...\")\n model.saver.restore(sess, tf.train.latest_checkpoint(ckpt_path))\n last_valid_cost, precision, recall, last_f1 = valid_epoch(data_valid_path, sess, model)\n print(' valid cost=%g; p=%g, r=%g, f1=%g' % (last_valid_cost, precision, recall, last_f1))\n sess.run(tf.variables_initializer(training_ops))\n else:\n print('Initializing Variables...')\n sess.run(tf.global_variables_initializer())\n\n print('3.Begin training...')\n print('max_epoch=%d, max_max_epoch=%d' % (FLAGS.max_epoch, FLAGS.max_max_epoch))\n for epoch in range(FLAGS.max_max_epoch):\n global_step = sess.run(model.global_step)\n print('Global step %d, lr=%g' % (global_step, sess.run(learning_rate)))\n if epoch == FLAGS.max_epoch: # update the embedding\n train_op = train_op1\n else:\n train_op = train_op2\n\n train_fetches = [merged, model.loss, train_op, update_op]\n valid_fetches = [merged, model.loss]\n train_epoch(data_train_path, sess, model, train_fetches, valid_fetches, train_writer, test_writer)\n # 最后再做一次验证\n valid_cost, precision, recall, f1 = valid_epoch(data_valid_path, sess, model)\n print('END.Global_step=%d: valid cost=%g; p=%g, r=%g, f1=%g' % (\n sess.run(model.global_step), valid_cost, precision, recall, f1))\n if f1 > last_f1: # save the better model\n saving_path = model.saver.save(sess, model_path, sess.run(model.global_step)+1)\n print('saved new model to %s ' % saving_path)\n\n\nif __name__ == '__main__':\n tf.app.run()\n" }, { "path": "zhihu-text-classification-master/models/wd_1_2_cnn_max/__init__.py", "content": "# -*- coding:utf-8 -*- \n\n" }, { "path": "zhihu-text-classification-master/models/wd_1_2_cnn_max/network.py", "content": "# -*- coding:utf-8 -*-\n\nimport tensorflow as tf\n\n\"\"\"wd_1_2_cnn_max\ntitle 部分使用 TextCNN；content 部分使用 TextCNN； 两部分输出按位取 max。\n\"\"\"\n\n\nclass Settings(object):\n def __init__(self):\n self.model_name = 'wd_1_2_cnn_max'\n self.title_len = 30\n self.content_len = 150\n self.filter_sizes = [2, 3, 4, 5, 7]\n self.n_filter = 256\n self.fc_hidden_size = 1024\n self.n_class = 1999\n self.summary_path = '../../summary/' + self.model_name + '/'\n self.ckpt_path = '../../ckpt/' + self.model_name + '/'\n\n\nclass TextCNN(object):\n \"\"\"\n title: inputs->textcnn->output_title\n content: inputs->textcnn->output_content\n max[output_title, output_content] -> fc+bn+relu -> sigmoid_entropy.\n \"\"\"\n\n def __init__(self, W_embedding, settings):\n self.model_name = settings.model_name\n self.title_len = settings.title_len\n self.content_len = settings.content_len\n self.filter_sizes = settings.filter_sizes\n self.n_filter = settings.n_filter\n self.n_filter_total = self.n_filter * len(self.filter_sizes)\n self.n_class = settings.n_class\n self.fc_hidden_size = settings.fc_hidden_size\n self._global_step = tf.Variable(0, trainable=False, name='Global_Step')\n self.update_emas = list()\n # placeholders\n self._tst = tf.placeholder(tf.bool)\n self._keep_prob = tf.placeholder(tf.float32, [])\n self._batch_size = tf.placeholder(tf.int32, [])\n\n with tf.name_scope('Inputs'):\n self._X1_inputs = tf.placeholder(tf.int64, [None, self.title_len], name='X1_inputs')\n self._X2_inputs = tf.placeholder(tf.int64, [None, self.content_len], name='X2_inputs')\n self._y_inputs = tf.placeholder(tf.float32, [None, self.n_class], name='y_input')\n\n with tf.variable_scope('embedding'):\n self.embedding = tf.get_variable(name='embedding', shape=W_embedding.shape,\n initializer=tf.constant_initializer(W_embedding), trainable=True)\n self.embedding_size = W_embedding.shape[1]\n\n with tf.variable_scope('cnn_text'):\n output_title = self.cnn_inference(self._X1_inputs, self.title_len)\n output_title = tf.expand_dims(output_title, 0)\n\n with tf.variable_scope('hcnn_content'):\n output_content = self.cnn_inference(self._X2_inputs, self.content_len)\n output_content = tf.expand_dims(output_content, 0)\n\n with tf.variable_scope('fc-bn-layer'):\n output = tf.concat([output_title, output_content], axis=0)\n output = tf.reduce_max(output, axis=0)\n W_fc = self.weight_variable([self.n_filter_total, self.fc_hidden_size], name='Weight_fc')\n tf.summary.histogram('W_fc', W_fc)\n h_fc = tf.matmul(output, W_fc, name='h_fc')\n beta_fc = tf.Variable(tf.constant(0.1, tf.float32, shape=[self.fc_hidden_size], name=\"beta_fc\"))\n tf.summary.histogram('beta_fc', beta_fc)\n fc_bn, update_ema_fc = self.batchnorm(h_fc, beta_fc, convolutional=False)\n self.update_emas.append(update_ema_fc)\n self.fc_bn_relu = tf.nn.relu(fc_bn, name=\"relu\")\n fc_bn_drop = tf.nn.dropout(self.fc_bn_relu, self.keep_prob)\n\n with tf.variable_scope('out_layer'):\n W_out = self.weight_variable([self.fc_hidden_size, self.n_class], name='Weight_out')\n tf.summary.histogram('Weight_out', W_out)\n b_out = self.bias_variable([self.n_class], name='bias_out')\n tf.summary.histogram('bias_out', b_out)\n self._y_pred = tf.nn.xw_plus_b(fc_bn_drop, W_out, b_out, name='y_pred') # 每个类别的分数 scores\n\n with tf.name_scope('loss'):\n self._loss = tf.reduce_mean(\n tf.nn.sigmoid_cross_entropy_with_logits(logits=self._y_pred, labels=self._y_inputs))\n tf.summary.scalar('loss', self._loss)\n\n self.saver = tf.train.Saver(max_to_keep=2)\n\n @property\n def tst(self):\n return self._tst\n\n @property\n def keep_prob(self):\n return self._keep_prob\n\n @property\n def batch_size(self):\n return self._batch_size\n\n @property\n def global_step(self):\n return self._global_step\n\n @property\n def X1_inputs(self):\n return self._X1_inputs\n\n @property\n def X2_inputs(self):\n return self._X2_inputs\n\n @property\n def y_inputs(self):\n return self._y_inputs\n\n @property\n def y_pred(self):\n return self._y_pred\n\n @property\n def loss(self):\n return self._loss\n\n def weight_variable(self, shape, name):\n \"\"\"Create a weight variable with appropriate initialization.\"\"\"\n initial = tf.truncated_normal(shape, stddev=0.1)\n return tf.Variable(initial, name=name)\n\n def bias_variable(self, shape, name):\n \"\"\"Create a bias variable with appropriate initialization.\"\"\"\n initial = tf.constant(0.1, shape=shape)\n return tf.Variable(initial, name=name)\n\n def batchnorm(self, Ylogits, offset, convolutional=False):\n \"\"\"batchnormalization.\n Args:\n Ylogits: 1D向量或者是3D的卷积结果。\n num_updates: 迭代的global_step\n offset：表示beta，全局均值；在 RELU 激活中一般初始化为 0.1。\n scale：表示lambda，全局方差；在 sigmoid 激活中需要，这 RELU 激活中作用不大。\n m: 表示batch均值；v:表示batch方差。\n bnepsilon：一个很小的浮点数，防止除以 0.\n Returns:\n Ybn: 和 Ylogits 的维度一样，就是经过 Batch Normalization 处理的结果。\n update_moving_everages：更新mean和variance，主要是给最后的 test 使用。\n \"\"\"\n exp_moving_avg = tf.train.ExponentialMovingAverage(0.999,\n self._global_step) # adding the iteration prevents from averaging across non-existing iterations\n bnepsilon = 1e-5\n if convolutional:\n mean, variance = tf.nn.moments(Ylogits, [0, 1, 2])\n else:\n mean, variance = tf.nn.moments(Ylogits, [0])\n update_moving_everages = exp_moving_avg.apply([mean, variance])\n m = tf.cond(self.tst, lambda: exp_moving_avg.average(mean), lambda: mean)\n v = tf.cond(self.tst, lambda: exp_moving_avg.average(variance), lambda: variance)\n Ybn = tf.nn.batch_normalization(Ylogits, m, v, offset, None, bnepsilon)\n return Ybn, update_moving_everages\n\n def cnn_inference(self, X_inputs, n_step):\n \"\"\"TextCNN 模型。\n Args:\n X_inputs: tensor.shape=(batch_size, n_step)\n Returns:\n title_outputs: tensor.shape=(batch_size, self.n_filter_total)\n \"\"\"\n inputs = tf.nn.embedding_lookup(self.embedding, X_inputs)\n inputs = tf.expand_dims(inputs, -1)\n pooled_outputs = list()\n for i, filter_size in enumerate(self.filter_sizes):\n with tf.variable_scope(\"conv-maxpool-%s\" % filter_size):\n # Convolution Layer\n filter_shape = [filter_size, self.embedding_size, 1, self.n_filter]\n W_filter = self.weight_variable(shape=filter_shape, name='W_filter')\n beta = self.bias_variable(shape=[self.n_filter], name='beta_filter')\n tf.summary.histogram('beta', beta)\n conv = tf.nn.conv2d(inputs, W_filter, strides=[1, 1, 1, 1], padding=\"VALID\", name=\"conv\")\n conv_bn, update_ema = self.batchnorm(conv, beta, convolutional=True) # 在激活层前面加 BN\n # Apply nonlinearity, batch norm scaling is not useful with relus\n # batch norm offsets are used instead of biases,使用 BN 层的 offset，不要 biases\n h = tf.nn.relu(conv_bn, name=\"relu\")\n # Maxpooling over the outputs\n pooled = tf.nn.max_pool(h, ksize=[1, n_step - filter_size + 1, 1, 1],\n strides=[1, 1, 1, 1], padding='VALID', name=\"pool\")\n pooled_outputs.append(pooled)\n self.update_emas.append(update_ema)\n h_pool = tf.concat(pooled_outputs, 3)\n h_pool_flat = tf.reshape(h_pool, [-1, self.n_filter_total])\n return h_pool_flat # shape = [batch_size, self.n_filter_total]\n\n\n# test the model\n# def test():\n# import numpy as np\n# print('Begin testing...')\n# settings = Settings()\n# W_embedding = np.random.randn(50, 10)\n# config = tf.ConfigProto()\n# config.gpu_options.allow_growth = True\n# batch_size = 128\n# with tf.Session(config=config) as sess:\n# model = TextCNN(W_embedding, settings)\n# optimizer = tf.train.AdamOptimizer(0.001)\n# train_op = optimizer.minimize(model.loss)\n# update_op = tf.group(*model.update_emas)\n# sess.run(tf.global_variables_initializer())\n# fetch = [model.loss, model.y_pred, train_op, update_op]\n# loss_list = list()\n# for i in xrange(100):\n# X1_batch = np.zeros((batch_size, 30), dtype=float)\n# X2_batch = np.zeros((batch_size, 150), dtype=float)\n# y_batch = np.zeros((batch_size, 1999), dtype=int)\n# _batch_size = len(y_batch)\n# feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n# model.batch_size: _batch_size, model.tst: False, model.keep_prob: 0.5}\n# loss, y_pred, _, _ = sess.run(fetch, feed_dict=feed_dict)\n# loss_list.append(loss)\n# print(i, loss)\n#\n# if __name__ == '__main__':\n# test()\n" }, { "path": "zhihu-text-classification-master/models/wd_1_2_cnn_max/predict.py", "content": "# -*- coding:utf-8 -*-\n\nfrom __future__ import print_function\nfrom __future__ import division\nimport tensorflow as tf\nimport numpy as np\nfrom tqdm import tqdm\nimport os\nimport sys\nimport time\nimport network\n\nsys.path.append('../..')\nfrom evaluator import score_eval\n\nsettings = network.Settings()\ntitle_len = settings.title_len\nmodel_name = settings.model_name\nckpt_path = settings.ckpt_path\n\nlocal_scores_path = '../../local_scores/'\nscores_path = '../../scores/'\nif not os.path.exists(local_scores_path):\n os.makedirs(local_scores_path)\nif not os.path.exists(scores_path):\n os.makedirs(scores_path)\n\nembedding_path = '../../data/word_embedding.npy'\ndata_valid_path = '../../data/wd-data/data_valid/'\ndata_test_path = '../../data/wd-data/data_test/'\nva_batches = os.listdir(data_valid_path)\nte_batches = os.listdir(data_test_path) # batch 文件名列表\nn_va_batches = len(va_batches)\nn_te_batches = len(te_batches)\n\n\ndef get_batch(batch_id):\n \"\"\"get a batch from valid data\"\"\"\n new_batch = np.load(data_valid_path + str(batch_id) + '.npz')\n X_batch = new_batch['X']\n y_batch = new_batch['y']\n X1_batch = X_batch[:, :title_len]\n X2_batch = X_batch[:, title_len:]\n return [X1_batch, X2_batch, y_batch]\n\n\ndef get_test_batch(batch_id):\n \"\"\"get a batch from test data\"\"\"\n X_batch = np.load(data_test_path + str(batch_id) + '.npy')\n X1_batch = X_batch[:, :title_len]\n X2_batch = X_batch[:, title_len:]\n return [X1_batch, X2_batch]\n\n\ndef local_predict(sess, model):\n \"\"\"Test on the valid data.\"\"\"\n time0 = time.time()\n predict_labels_list = list() # 所有的预测结果\n marked_labels_list = list()\n predict_scores = list()\n for i in tqdm(xrange(n_va_batches)):\n [X1_batch, X2_batch, y_batch] = get_batch(i)\n marked_labels_list.extend(y_batch)\n _batch_size = len(X1_batch)\n fetches = [model.y_pred]\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n predict_labels = sess.run(fetches, feed_dict)[0]\n predict_scores.append(predict_labels)\n predict_labels = list(map(lambda label: label.argsort()[-1:-6:-1], predict_labels)) # 取最大的5个下标\n predict_labels_list.extend(predict_labels)\n predict_label_and_marked_label_list = zip(predict_labels_list, marked_labels_list)\n precision, recall, f1 = score_eval(predict_label_and_marked_label_list)\n print('Local valid p=%g, r=%g, f1=%g' % (precision, recall, f1))\n predict_scores = np.vstack(np.asarray(predict_scores))\n local_scores_name = local_scores_path + model_name + '.npy'\n np.save(local_scores_name, predict_scores)\n print('local_scores.shape=', predict_scores.shape)\n print('Writed the scores into %s, time %g s' % (local_scores_name, time.time() - time0))\n\n\ndef predict(sess, model):\n \"\"\"Test on the test data.\"\"\"\n time0 = time.time()\n predict_scores = list()\n for i in tqdm(xrange(n_te_batches)):\n [X1_batch, X2_batch] = get_test_batch(i)\n _batch_size = len(X1_batch)\n fetches = [model.y_pred]\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n predict_labels = sess.run(fetches, feed_dict)[0]\n predict_scores.append(predict_labels)\n predict_scores = np.vstack(np.asarray(predict_scores))\n scores_name = scores_path + model_name + '.npy'\n np.save(scores_name, predict_scores)\n print('scores.shape=', predict_scores.shape)\n print('Writed the scores into %s, time %g s' % (scores_name, time.time() - time0))\n\n\ndef main(_):\n if not os.path.exists(ckpt_path + 'checkpoint'):\n print('there is not saved model, please check the ckpt path')\n exit()\n print('Loading model...')\n W_embedding = np.load(embedding_path)\n config = tf.ConfigProto()\n config.gpu_options.allow_growth = True\n with tf.Session(config=config) as sess:\n model = network.TextCNN(W_embedding, settings)\n model.saver.restore(sess, tf.train.latest_checkpoint(ckpt_path))\n print('Local predicting...')\n local_predict(sess, model)\n print('Test predicting...')\n predict(sess, model)\n\n\nif __name__ == '__main__':\n tf.app.run()\n" }, { "path": "zhihu-text-classification-master/models/wd_1_2_cnn_max/train.py", "content": "# -*- coding:utf-8 -*-\n\nfrom __future__ import print_function\nfrom __future__ import division\nimport tensorflow as tf\nimport numpy as np\nfrom tqdm import tqdm\nimport os\nimport sys\nimport shutil\nimport time\nimport network\n\nsys.path.append('../..')\nfrom data_helpers import to_categorical\nfrom evaluator import score_eval\n\nflags = tf.flags\nflags.DEFINE_bool('is_retrain', False, 'if is_retrain is true, not rebuild the summary')\nflags.DEFINE_integer('max_epoch', 1, 'update the embedding after max_epoch, default: 1')\nflags.DEFINE_integer('max_max_epoch', 6, 'all training epoches, default: 6')\nflags.DEFINE_float('lr', 1e-3, 'initial learning rate, default: 1e-3')\nflags.DEFINE_float('decay_rate', 0.65, 'decay rate, default: 0.65')\nflags.DEFINE_float('keep_prob', 0.5, 'keep_prob for training, default: 0.5')\n# 正式\n\nflags.DEFINE_integer('decay_step', 15000, 'decay_step, default: 15000')\nflags.DEFINE_integer('valid_step', 10000, 'valid_step, default: 10000')\nflags.DEFINE_float('last_f1', 0.35, 'if valid_f1 > last_f1, save new model. default: 0.40')\n\n# 测试\n# flags.DEFINE_integer('decay_step', 1000, 'decay_step, default: 1000')\n# flags.DEFINE_integer('valid_step', 500, 'valid_step, default: 500')\n# flags.DEFINE_float('last_f1', 0.10, 'if valid_f1 > last_f1, save new model. default: 0.10')\nFLAGS = flags.FLAGS\n\nlr = FLAGS.lr\nlast_f1 = FLAGS.last_f1\nsettings = network.Settings()\ntitle_len = settings.title_len\nsummary_path = settings.summary_path\nckpt_path = settings.ckpt_path\nmodel_path = ckpt_path + 'model.ckpt'\n\nembedding_path = '../../data/word_embedding.npy'\ndata_train_path = '../../data/wd-data/data_train/'\ndata_valid_path = '../../data/wd-data/data_valid/'\ntr_batches = os.listdir(data_train_path) # batch 文件名列表\nva_batches = os.listdir(data_valid_path)\nn_tr_batches = len(tr_batches)\nn_va_batches = len(va_batches)\n\n# 测试\n# n_tr_batches = 1000\n# n_va_batches = 50\n\n\ndef get_batch(data_path, batch_id):\n \"\"\"get a batch from data_path\"\"\"\n new_batch = np.load(data_path + str(batch_id) + '.npz')\n X_batch = new_batch['X']\n y_batch = new_batch['y']\n X1_batch = X_batch[:, :title_len]\n X2_batch = X_batch[:, title_len:]\n return [X1_batch, X2_batch, y_batch]\n\n\ndef valid_epoch(data_path, sess, model):\n \"\"\"Test on the valid data.\"\"\"\n va_batches = os.listdir(data_path)\n n_va_batches = len(va_batches)\n _costs = 0.0\n predict_labels_list = list() # 所有的预测结果\n marked_labels_list = list()\n for i in range(n_va_batches):\n [X1_batch, X2_batch, y_batch] = get_batch(data_path, i)\n marked_labels_list.extend(y_batch)\n y_batch = to_categorical(y_batch)\n _batch_size = len(y_batch)\n fetches = [model.loss, model.y_pred]\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n _cost, predict_labels = sess.run(fetches, feed_dict)\n _costs += _cost\n predict_labels = list(map(lambda label: label.argsort()[-1:-6:-1], predict_labels)) # 取最大的5个下标\n predict_labels_list.extend(predict_labels)\n predict_label_and_marked_label_list = zip(predict_labels_list, marked_labels_list)\n precision, recall, f1 = score_eval(predict_label_and_marked_label_list)\n mean_cost = _costs / n_va_batches\n return mean_cost, precision, recall, f1\n\n\ndef train_epoch(data_path, sess, model, train_fetches, valid_fetches, train_writer, test_writer):\n global last_f1\n global lr\n time0 = time.time()\n batch_indexs = np.random.permutation(n_tr_batches) # shuffle the training data\n for batch in tqdm(range(n_tr_batches)):\n global_step = sess.run(model.global_step)\n if 0 == (global_step + 1) % FLAGS.valid_step:\n valid_cost, precision, recall, f1 = valid_epoch(data_valid_path, sess, model)\n print('Global_step=%d: valid cost=%g; p=%g, r=%g, f1=%g, time=%g s' % (\n global_step, valid_cost, precision, recall, f1, time.time() - time0))\n time0 = time.time()\n if f1 > last_f1:\n last_f1 = f1\n saving_path = model.saver.save(sess, model_path, global_step+1)\n print('saved new model to %s ' % saving_path)\n # training\n batch_id = batch_indexs[batch]\n [X1_batch, X2_batch, y_batch] = get_batch(data_train_path, batch_id)\n y_batch = to_categorical(y_batch)\n _batch_size = len(y_batch)\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: False, model.keep_prob: FLAGS.keep_prob}\n summary, _cost, _, _ = sess.run(train_fetches, feed_dict) # the cost is the mean cost of one batch\n # valid per 500 steps\n if 0 == (global_step + 1) % 500:\n train_writer.add_summary(summary, global_step)\n batch_id = np.random.randint(0, n_va_batches) # 随机选一个验证batch\n [X1_batch, X2_batch, y_batch] = get_batch(data_valid_path, batch_id)\n y_batch = to_categorical(y_batch)\n _batch_size = len(y_batch)\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n summary, _cost = sess.run(valid_fetches, feed_dict)\n test_writer.add_summary(summary, global_step)\n\n\ndef main(_):\n global ckpt_path\n global last_f1\n if not os.path.exists(ckpt_path):\n os.makedirs(ckpt_path)\n if not os.path.exists(summary_path):\n os.makedirs(summary_path)\n elif not FLAGS.is_retrain: # 重新训练本模型，删除以前的 summary\n shutil.rmtree(summary_path)\n os.makedirs(summary_path)\n if not os.path.exists(summary_path):\n os.makedirs(summary_path)\n\n print('1.Loading data...')\n W_embedding = np.load(embedding_path)\n print('training sample_num = %d' % n_tr_batches)\n print('valid sample_num = %d' % n_va_batches)\n\n # Initial or restore the model\n print('2.Building model...')\n config = tf.ConfigProto()\n config.gpu_options.allow_growth = True\n with tf.Session(config=config) as sess:\n model = network.TextCNN(W_embedding, settings)\n with tf.variable_scope('training_ops') as vs:\n learning_rate = tf.train.exponential_decay(FLAGS.lr, model.global_step, FLAGS.decay_step,\n FLAGS.decay_rate, staircase=True)\n # two optimizer: op1, update embedding; op2, do not update embedding.\n with tf.variable_scope('Optimizer1'):\n tvars1 = tf.trainable_variables()\n grads1 = tf.gradients(model.loss, tvars1)\n optimizer1 = tf.train.AdamOptimizer(learning_rate=learning_rate)\n train_op1 = optimizer1.apply_gradients(zip(grads1, tvars1),\n global_step=model.global_step)\n with tf.variable_scope('Optimizer2'):\n tvars2 = [tvar for tvar in tvars1 if 'embedding' not in tvar.name]\n grads2 = tf.gradients(model.loss, tvars2)\n optimizer2 = tf.train.AdamOptimizer(learning_rate=learning_rate)\n train_op2 = optimizer2.apply_gradients(zip(grads2, tvars2),\n global_step=model.global_step)\n update_op = tf.group(*model.update_emas)\n merged = tf.summary.merge_all() # summary\n train_writer = tf.summary.FileWriter(summary_path + 'train', sess.graph)\n test_writer = tf.summary.FileWriter(summary_path + 'test')\n training_ops = [v for v in tf.global_variables() if v.name.startswith(vs.name+'/')]\n\n # 如果已经保存过模型，导入上次的模型\n if os.path.exists(ckpt_path + \"checkpoint\"):\n print(\"Restoring Variables from Checkpoint...\")\n model.saver.restore(sess, tf.train.latest_checkpoint(ckpt_path))\n last_valid_cost, precision, recall, last_f1 = valid_epoch(data_valid_path, sess, model)\n print(' valid cost=%g; p=%g, r=%g, f1=%g' % (last_valid_cost, precision, recall, last_f1))\n sess.run(tf.variables_initializer(training_ops))\n train_op2 = train_op1\n else:\n print('Initializing Variables...')\n sess.run(tf.global_variables_initializer())\n\n print('3.Begin training...')\n print('max_epoch=%d, max_max_epoch=%d' % (FLAGS.max_epoch, FLAGS.max_max_epoch))\n train_op = train_op2\n for epoch in range(FLAGS.max_max_epoch):\n global_step = sess.run(model.global_step)\n print('Global step %d, lr=%g' % (global_step, sess.run(learning_rate)))\n if epoch == FLAGS.max_epoch: # update the embedding\n train_op = train_op1\n train_fetches = [merged, model.loss, train_op, update_op]\n valid_fetches = [merged, model.loss]\n train_epoch(data_train_path, sess, model, train_fetches, valid_fetches, train_writer, test_writer)\n # 最后再做一次验证\n valid_cost, precision, recall, f1 = valid_epoch(data_valid_path, sess, model)\n print('END.Global_step=%d: valid cost=%g; p=%g, r=%g, f1=%g' % (\n sess.run(model.global_step), valid_cost, precision, recall, f1))\n if f1 > last_f1: # save the better model\n saving_path = model.saver.save(sess, model_path, sess.run(model.global_step)+1)\n print('saved new model to %s ' % saving_path)\n\n\nif __name__ == '__main__':\n tf.app.run()\n" }, { "path": "zhihu-text-classification-master/models/wd_2_hcnn/__init__.py", "content": "# -*- coding:utf-8 -*- \n\n" }, { "path": "zhihu-text-classification-master/models/wd_2_hcnn/network.py", "content": "# -*- coding:utf-8 -*-\n\nimport tensorflow as tf\n\n\"\"\"wd_2_hcnn\ntitle 部分使用 TextCNN；content 部分使用分层的 TextCNN。\n\"\"\"\n\n\nclass Settings(object):\n def __init__(self):\n self.model_name = 'wd_2_hcnn'\n self.title_len = self.sent_len = 30\n self.doc_len = 10\n self.sent_filter_sizes = [2, 3, 4, 5]\n self.doc_filter_sizes = [2, 3, 4]\n self.n_filter = 256\n self.fc_hidden_size = 1024\n self.n_class = 1999\n self.summary_path = '../../summary/' + self.model_name + '/'\n self.ckpt_path = '../../ckpt/' + self.model_name + '/'\n\n\nclass HCNN(object):\n \"\"\"\n title: inputs->textcnn->output_title\n content: inputs->hcnn->output_content\n concat[output_title, output_content] -> fc+bn+relu -> sigmoid_entropy.\n \"\"\"\n\n def __init__(self, W_embedding, settings):\n self.model_name = settings.model_name\n self.sent_len = settings.sent_len\n self.doc_len = settings.doc_len\n self.sent_filter_sizes = settings.sent_filter_sizes\n self.doc_filter_sizes = settings.doc_filter_sizes\n self.n_filter = settings.n_filter\n self.n_class = settings.n_class\n self.fc_hidden_size = settings.fc_hidden_size\n self._global_step = tf.Variable(0, trainable=False, name='Global_Step')\n self.update_emas = list()\n # placeholders\n self._tst = tf.placeholder(tf.bool)\n self._keep_prob = tf.placeholder(tf.float32, [])\n self._batch_size = tf.placeholder(tf.int32, [])\n\n with tf.name_scope('Inputs'):\n self._X1_inputs = tf.placeholder(tf.int64, [None, self.sent_len], name='X1_inputs')\n self._X2_inputs = tf.placeholder(tf.int64, [None, self.doc_len * self.sent_len], name='X2_inputs')\n self._y_inputs = tf.placeholder(tf.float32, [None, self.n_class], name='y_input')\n\n with tf.variable_scope('embedding'):\n self.embedding = tf.get_variable(name='embedding', shape=W_embedding.shape,\n initializer=tf.constant_initializer(W_embedding), trainable=True)\n self.embedding_size = W_embedding.shape[1]\n\n with tf.variable_scope('cnn_text'):\n output_title = self.cnn_inference(self._X1_inputs)\n\n with tf.variable_scope('hcnn_content'):\n output_content = self.hcnn_inference(self._X2_inputs)\n\n with tf.variable_scope('fc-bn-layer'):\n output = tf.concat([output_title, output_content], axis=1)\n output_size = self.n_filter * (len(self.sent_filter_sizes) + len(self.doc_filter_sizes))\n W_fc = self.weight_variable([output_size, self.fc_hidden_size], name='Weight_fc')\n tf.summary.histogram('W_fc', W_fc)\n h_fc = tf.matmul(output, W_fc, name='h_fc')\n beta_fc = tf.Variable(tf.constant(0.1, tf.float32, shape=[self.fc_hidden_size], name=\"beta_fc\"))\n tf.summary.histogram('beta_fc', beta_fc)\n fc_bn, update_ema_fc = self.batchnorm(h_fc, beta_fc, convolutional=False)\n self.update_emas.append(update_ema_fc)\n self.fc_bn_relu = tf.nn.relu(fc_bn, name=\"relu\")\n fc_bn_drop = tf.nn.dropout(self.fc_bn_relu, self.keep_prob)\n\n with tf.variable_scope('out_layer'):\n W_out = self.weight_variable([self.fc_hidden_size, self.n_class], name='Weight_out')\n tf.summary.histogram('Weight_out', W_out)\n b_out = self.bias_variable([self.n_class], name='bias_out')\n tf.summary.histogram('bias_out', b_out)\n self._y_pred = tf.nn.xw_plus_b(fc_bn_drop, W_out, b_out, name='y_pred') # 每个类别的分数 scores\n\n with tf.name_scope('loss'):\n self._loss = tf.reduce_mean(\n tf.nn.sigmoid_cross_entropy_with_logits(logits=self._y_pred, labels=self._y_inputs))\n tf.summary.scalar('loss', self._loss)\n\n self.saver = tf.train.Saver(max_to_keep=2)\n\n @property\n def tst(self):\n return self._tst\n\n @property\n def keep_prob(self):\n return self._keep_prob\n\n @property\n def batch_size(self):\n return self._batch_size\n\n @property\n def global_step(self):\n return self._global_step\n\n @property\n def X1_inputs(self):\n return self._X1_inputs\n\n @property\n def X2_inputs(self):\n return self._X2_inputs\n\n @property\n def y_inputs(self):\n return self._y_inputs\n\n @property\n def y_pred(self):\n return self._y_pred\n\n @property\n def loss(self):\n return self._loss\n\n def weight_variable(self, shape, name):\n \"\"\"Create a weight variable with appropriate initialization.\"\"\"\n initial = tf.truncated_normal(shape, stddev=0.1)\n return tf.Variable(initial, name=name)\n\n def bias_variable(self, shape, name):\n \"\"\"Create a bias variable with appropriate initialization.\"\"\"\n initial = tf.constant(0.1, shape=shape)\n return tf.Variable(initial, name=name)\n\n def batchnorm(self, Ylogits, offset, convolutional=False):\n \"\"\"batchnormalization.\n Args:\n Ylogits: 1D向量或者是3D的卷积结果。\n num_updates: 迭代的global_step\n offset：表示beta，全局均值；在 RELU 激活中一般初始化为 0.1。\n scale：表示lambda，全局方差；在 sigmoid 激活中需要，这 RELU 激活中作用不大。\n m: 表示batch均值；v:表示batch方差。\n bnepsilon：一个很小的浮点数，防止除以 0.\n Returns:\n Ybn: 和 Ylogits 的维度一样，就是经过 Batch Normalization 处理的结果。\n update_moving_everages：更新mean和variance，主要是给最后的 test 使用。\n \"\"\"\n exp_moving_avg = tf.train.ExponentialMovingAverage(0.999,\n self._global_step) # adding the iteration prevents from averaging across non-existing iterations\n bnepsilon = 1e-5\n if convolutional:\n mean, variance = tf.nn.moments(Ylogits, [0, 1, 2])\n else:\n mean, variance = tf.nn.moments(Ylogits, [0])\n update_moving_everages = exp_moving_avg.apply([mean, variance])\n m = tf.cond(self.tst, lambda: exp_moving_avg.average(mean), lambda: mean)\n v = tf.cond(self.tst, lambda: exp_moving_avg.average(variance), lambda: variance)\n Ybn = tf.nn.batch_normalization(Ylogits, m, v, offset, None, bnepsilon)\n return Ybn, update_moving_everages\n\n def textcnn(self, X_inputs, n_step, filter_sizes, embed_size):\n \"\"\"build the TextCNN network.\n n_step: the sentence len.\"\"\"\n inputs = tf.expand_dims(X_inputs, -1)\n pooled_outputs = list()\n for i, filter_size in enumerate(filter_sizes):\n with tf.name_scope(\"conv-maxpool-%s\" % filter_size):\n # Convolution Layer\n filter_shape = [filter_size, embed_size, 1, self.n_filter]\n W_filter = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name=\"W_filter\")\n beta = tf.Variable(tf.constant(0.1, tf.float32, shape=[self.n_filter], name=\"beta\"))\n tf.summary.histogram('beta', beta)\n conv = tf.nn.conv2d(inputs, W_filter, strides=[1, 1, 1, 1], padding=\"VALID\", name=\"conv\")\n conv_bn, update_ema = self.batchnorm(conv, beta, convolutional=True) # 在激活层前面加 BN\n # Apply nonlinearity, batch norm scaling is not useful with relus\n # batch norm offsets are used instead of biases,使用 BN 层的 offset，不要 biases\n h = tf.nn.relu(conv_bn, name=\"relu\")\n # Maxpooling over the outputs\n pooled = tf.nn.max_pool(h, ksize=[1, n_step - filter_size + 1, 1, 1],\n strides=[1, 1, 1, 1], padding='VALID', name=\"pool\")\n pooled_outputs.append(pooled)\n self.update_emas.append(update_ema)\n h_pool = tf.concat(pooled_outputs, 3)\n n_filter_total = self.n_filter * len(filter_sizes)\n h_pool_flat = tf.reshape(h_pool, [-1, n_filter_total])\n return h_pool_flat # shape = [-1, n_filter_total]\n\n def cnn_inference(self, X_inputs):\n \"\"\"TextCNN 模型。title部分。\n Args:\n X_inputs: tensor.shape=(batch_size, title_len)\n Returns:\n title_outputs: tensor.shape=(batch_size, n_filter*filter_num_sent)\n \"\"\"\n inputs = tf.nn.embedding_lookup(self.embedding, X_inputs)\n with tf.variable_scope('title_encoder'): # 生成 title 的向量表示\n title_outputs = self.textcnn(inputs, self.sent_len, self.sent_filter_sizes, embed_size=self.embedding_size)\n return title_outputs # shape = [batch_size, n_filter*filter_num_sent]\n\n def hcnn_inference(self, X_inputs):\n \"\"\"分层 TextCNN 模型。content部分。\n Args:\n X_inputs: tensor.shape=(batch_size, doc_len*sent_len)\n Returns:\n doc_attn_outputs: tensor.shape=(batch_size, n_filter*filter_num_doc)\n \"\"\"\n inputs = tf.nn.embedding_lookup(self.embedding,\n X_inputs) # inputs.shape=[batch_size, doc_len*sent_len, embedding_size]\n sent_inputs = tf.reshape(inputs, [self.batch_size * self.doc_len, self.sent_len,\n self.embedding_size]) # [batch_size*doc_len, sent_len, embedding_size]\n with tf.variable_scope('sentence_encoder'): # 生成句向量\n sent_outputs = self.textcnn(sent_inputs, self.sent_len, self.sent_filter_sizes, self.embedding_size)\n with tf.variable_scope('doc_encoder'): # 生成文档向量\n doc_inputs = tf.reshape(sent_outputs, [self.batch_size, self.doc_len, self.n_filter * len(\n self.sent_filter_sizes)]) # [batch_size, doc_len, n_filter*len(filter_sizes_sent)]\n doc_outputs = self.textcnn(doc_inputs, self.doc_len, self.doc_filter_sizes, self.n_filter * len(\n self.sent_filter_sizes)) # [batch_size, doc_len, n_filter*filter_num_doc]\n return doc_outputs # [batch_size, n_filter*len(doc_filter_sizes)]\n\n# test the model\n# def test():\n# import numpy as np\n# print('Begin testing...')\n# settings = Settings()\n# W_embedding = np.random.randn(50, 10)\n# config = tf.ConfigProto()\n# config.gpu_options.allow_growth = True\n# batch_size = 128\n# with tf.Session(config=config) as sess:\n# model = HCNN(W_embedding, settings)\n# optimizer = tf.train.AdamOptimizer(0.001)\n# train_op = optimizer.minimize(model.loss)\n# update_op = tf.group(*model.update_emas)\n# sess.run(tf.global_variables_initializer())\n# fetch = [model.loss, model.y_pred, train_op, update_op]\n# loss_list = list()\n# for i in xrange(100):\n# X1_batch = np.zeros((batch_size, 30), dtype=float)\n# X2_batch = np.zeros((batch_size, 10 * 30), dtype=float)\n# y_batch = np.zeros((batch_size, 1999), dtype=int)\n# _batch_size = len(y_batch)\n# feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n# model.batch_size: _batch_size, model.tst: False, model.keep_prob: 0.5}\n# loss, y_pred, _, _ = sess.run(fetch, feed_dict=feed_dict)\n# loss_list.append(loss)\n# print(i, loss)\n\n# test()\n" }, { "path": "zhihu-text-classification-master/models/wd_2_hcnn/predict.py", "content": "# -*- coding:utf-8 -*-\n\nfrom __future__ import print_function\nfrom __future__ import division\nimport tensorflow as tf\nimport numpy as np\nfrom tqdm import tqdm\nimport os\nimport sys\nimport time\nimport network\n\nsys.path.append('../..')\nfrom evaluator import score_eval\n\nsettings = network.Settings()\ntitle_len = settings.title_len\nmodel_name = settings.model_name\nckpt_path = settings.ckpt_path\n\nlocal_scores_path = '../../local_scores/'\nscores_path = '../../scores/'\nif not os.path.exists(local_scores_path):\n os.makedirs(local_scores_path)\nif not os.path.exists(scores_path):\n os.makedirs(scores_path)\n\nembedding_path = '../../data/word_embedding.npy'\ndata_valid_path = '../../data/wd-data/seg_valid/'\ndata_test_path = '../../data/wd-data/seg_test/'\nva_batches = os.listdir(data_valid_path)\nte_batches = os.listdir(data_test_path) # batch 文件名列表\nn_va_batches = len(va_batches)\nn_te_batches = len(te_batches)\n\n\ndef get_batch(batch_id):\n \"\"\"get a batch from valid data\"\"\"\n new_batch = np.load(data_valid_path + str(batch_id) + '.npz')\n X_batch = new_batch['X']\n y_batch = new_batch['y']\n X1_batch = X_batch[:, :title_len]\n X2_batch = X_batch[:, title_len:]\n return [X1_batch, X2_batch, y_batch]\n\n\ndef get_test_batch(batch_id):\n \"\"\"get a batch from test data\"\"\"\n X_batch = np.load(data_test_path + str(batch_id) + '.npy')\n X1_batch = X_batch[:, :title_len]\n X2_batch = X_batch[:, title_len:]\n return [X1_batch, X2_batch]\n\n\ndef local_predict(sess, model):\n \"\"\"Test on the valid data.\"\"\"\n time0 = time.time()\n predict_labels_list = list() # 所有的预测结果\n marked_labels_list = list()\n predict_scores = list()\n for i in tqdm(xrange(n_va_batches)):\n [X1_batch, X2_batch, y_batch] = get_batch(i)\n marked_labels_list.extend(y_batch)\n _batch_size = len(X1_batch)\n fetches = [model.y_pred]\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n predict_labels = sess.run(fetches, feed_dict)[0]\n predict_scores.append(predict_labels)\n predict_labels = list(map(lambda label: label.argsort()[-1:-6:-1], predict_labels)) # 取最大的5个下标\n predict_labels_list.extend(predict_labels)\n predict_label_and_marked_label_list = zip(predict_labels_list, marked_labels_list)\n precision, recall, f1 = score_eval(predict_label_and_marked_label_list)\n print('Local valid p=%g, r=%g, f1=%g' % (precision, recall, f1))\n predict_scores = np.vstack(np.asarray(predict_scores))\n local_scores_name = local_scores_path + model_name + '.npy'\n np.save(local_scores_name, predict_scores)\n print('local_scores.shape=', predict_scores.shape)\n print('Writed the scores into %s, time %g s' % (local_scores_name, time.time() - time0))\n\n\ndef predict(sess, model):\n \"\"\"Test on the test data.\"\"\"\n time0 = time.time()\n predict_scores = list()\n for i in tqdm(xrange(n_te_batches)):\n [X1_batch, X2_batch] = get_test_batch(i)\n _batch_size = len(X1_batch)\n fetches = [model.y_pred]\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n predict_labels = sess.run(fetches, feed_dict)[0]\n predict_scores.append(predict_labels)\n predict_scores = np.vstack(np.asarray(predict_scores))\n scores_name = scores_path + model_name + '.npy'\n np.save(scores_name, predict_scores)\n print('scores.shape=', predict_scores.shape)\n print('Writed the scores into %s, time %g s' % (scores_name, time.time() - time0))\n\n\ndef main(_):\n if not os.path.exists(ckpt_path + 'checkpoint'):\n print('there is not saved model, please check the ckpt path')\n exit()\n print('Loading model...')\n W_embedding = np.load(embedding_path)\n config = tf.ConfigProto()\n config.gpu_options.allow_growth = True\n with tf.Session(config=config) as sess:\n model = network.HCNN(W_embedding, settings)\n model.saver.restore(sess, tf.train.latest_checkpoint(ckpt_path))\n print('Local predicting...')\n local_predict(sess, model)\n print('Test predicting...')\n predict(sess, model)\n\n\nif __name__ == '__main__':\n tf.app.run()\n" }, { "path": "zhihu-text-classification-master/models/wd_2_hcnn/train.py", "content": "# -*- coding:utf-8 -*-\n\nfrom __future__ import print_function\nfrom __future__ import division\nimport tensorflow as tf\nimport numpy as np\nfrom tqdm import tqdm\nimport os\nimport sys\nimport shutil\nimport time\nimport network\n\nsys.path.append('../..')\nfrom data_helpers import to_categorical\nfrom evaluator import score_eval\n\nflags = tf.flags\nflags.DEFINE_bool('is_retrain', False, 'if is_retrain is true, not rebuild the summary')\nflags.DEFINE_integer('max_epoch', 1, 'update the embedding after max_epoch, default: 1')\nflags.DEFINE_integer('max_max_epoch', 6, 'all training epoches, default: 6')\nflags.DEFINE_float('lr', 1e-3, 'initial learning rate, default: 1e-3')\nflags.DEFINE_float('decay_rate', 0.65, 'decay rate, default: 0.65')\nflags.DEFINE_float('keep_prob', 0.5, 'keep_prob for training, default: 0.5')\n# 正式\nflags.DEFINE_integer('decay_step', 15000, 'decay_step, default: 15000')\nflags.DEFINE_integer('valid_step', 10000, 'valid_step, default: 10000')\nflags.DEFINE_float('last_f1', 0.38, 'if valid_f1 > last_f1, save new model. default: 0.40')\n\n# 测试\n# flags.DEFINE_integer('decay_step', 1000, 'decay_step, default: 1000')\n# flags.DEFINE_integer('valid_step', 500, 'valid_step, default: 500')\n# flags.DEFINE_float('last_f1', 0.10, 'if valid_f1 > last_f1, save new model. default: 0.10')\nFLAGS = flags.FLAGS\n\nlr = FLAGS.lr\nlast_f1 = FLAGS.last_f1\nsettings = network.Settings()\ntitle_len = settings.title_len\nsummary_path = settings.summary_path\nckpt_path = settings.ckpt_path\nmodel_path = ckpt_path + 'model.ckpt'\n\nembedding_path = '../../data/word_embedding.npy'\ndata_train_path = '../../data/wd-data/seg_train/'\ndata_valid_path = '../../data/wd-data/seg_valid/'\ntr_batches = os.listdir(data_train_path) # batch 文件名列表\nva_batches = os.listdir(data_valid_path)\nn_tr_batches = len(tr_batches)\nn_va_batches = len(va_batches)\n\n# 测试\n# n_tr_batches = 1000\n# n_va_batches = 50\n\n\ndef get_batch(data_path, batch_id):\n \"\"\"get a batch from data_path\"\"\"\n new_batch = np.load(data_path + str(batch_id) + '.npz')\n X_batch = new_batch['X']\n y_batch = new_batch['y']\n X1_batch = X_batch[:, :title_len]\n X2_batch = X_batch[:, title_len:]\n return [X1_batch, X2_batch, y_batch]\n\n\ndef valid_epoch(data_path, sess, model):\n \"\"\"Test on the valid data.\"\"\"\n va_batches = os.listdir(data_path)\n n_va_batches = len(va_batches)\n _costs = 0.0\n predict_labels_list = list() # 所有的预测结果\n marked_labels_list = list()\n for i in range(n_va_batches):\n [X1_batch, X2_batch, y_batch] = get_batch(data_path, i)\n marked_labels_list.extend(y_batch)\n y_batch = to_categorical(y_batch)\n _batch_size = len(y_batch)\n fetches = [model.loss, model.y_pred]\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n _cost, predict_labels = sess.run(fetches, feed_dict)\n _costs += _cost\n predict_labels = list(map(lambda label: label.argsort()[-1:-6:-1], predict_labels)) # 取最大的5个下标\n predict_labels_list.extend(predict_labels)\n predict_label_and_marked_label_list = zip(predict_labels_list, marked_labels_list)\n precision, recall, f1 = score_eval(predict_label_and_marked_label_list)\n mean_cost = _costs / n_va_batches\n return mean_cost, precision, recall, f1\n\n\ndef train_epoch(data_path, sess, model, train_fetches, valid_fetches, train_writer, test_writer):\n global last_f1\n global lr\n time0 = time.time()\n batch_indexs = np.random.permutation(n_tr_batches) # shuffle the training data\n for batch in tqdm(range(n_tr_batches)):\n global_step = sess.run(model.global_step)\n if 0 == (global_step + 1) % FLAGS.valid_step:\n valid_cost, precision, recall, f1 = valid_epoch(data_valid_path, sess, model)\n print('Global_step=%d: valid cost=%g; p=%g, r=%g, f1=%g, time=%g s' % (\n global_step, valid_cost, precision, recall, f1, time.time() - time0))\n time0 = time.time()\n if f1 > last_f1:\n last_f1 = f1\n saving_path = model.saver.save(sess, model_path, global_step+1)\n print('saved new model to %s ' % saving_path)\n # training\n batch_id = batch_indexs[batch]\n [X1_batch, X2_batch, y_batch] = get_batch(data_train_path, batch_id)\n y_batch = to_categorical(y_batch)\n _batch_size = len(y_batch)\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: False, model.keep_prob: FLAGS.keep_prob}\n summary, _cost, _, _ = sess.run(train_fetches, feed_dict) # the cost is the mean cost of one batch\n # valid per 500 steps\n if 0 == (global_step + 1) % 500:\n train_writer.add_summary(summary, global_step)\n batch_id = np.random.randint(0, n_va_batches) # 随机选一个验证batch\n [X1_batch, X2_batch, y_batch] = get_batch(data_valid_path, batch_id)\n y_batch = to_categorical(y_batch)\n _batch_size = len(y_batch)\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n summary, _cost = sess.run(valid_fetches, feed_dict)\n test_writer.add_summary(summary, global_step)\n\n\ndef main(_):\n global ckpt_path\n global last_f1\n if not os.path.exists(ckpt_path):\n os.makedirs(ckpt_path)\n if not os.path.exists(summary_path):\n os.makedirs(summary_path)\n elif not FLAGS.is_retrain: # 重新训练本模型，删除以前的 summary\n shutil.rmtree(summary_path)\n os.makedirs(summary_path)\n if not os.path.exists(summary_path):\n os.makedirs(summary_path)\n\n print('1.Loading data...')\n W_embedding = np.load(embedding_path)\n print('training sample_num = %d' % n_tr_batches)\n print('valid sample_num = %d' % n_va_batches)\n\n # Initial or restore the model\n print('2.Building model...')\n config = tf.ConfigProto()\n config.gpu_options.allow_growth = True\n with tf.Session(config=config) as sess:\n model = network.HCNN(W_embedding, settings)\n with tf.variable_scope('training_ops') as vs:\n learning_rate = tf.train.exponential_decay(FLAGS.lr, model.global_step, FLAGS.decay_step,\n FLAGS.decay_rate, staircase=True)\n # two optimizer: op1, update embedding; op2, do not update embedding.\n with tf.variable_scope('Optimizer1'):\n tvars1 = tf.trainable_variables()\n grads1 = tf.gradients(model.loss, tvars1)\n optimizer1 = tf.train.AdamOptimizer(learning_rate=learning_rate)\n train_op1 = optimizer1.apply_gradients(zip(grads1, tvars1),\n global_step=model.global_step)\n with tf.variable_scope('Optimizer2'):\n tvars2 = [tvar for tvar in tvars1 if 'embedding' not in tvar.name]\n grads2 = tf.gradients(model.loss, tvars2)\n optimizer2 = tf.train.AdamOptimizer(learning_rate=learning_rate)\n train_op2 = optimizer2.apply_gradients(zip(grads2, tvars2),\n global_step=model.global_step)\n update_op = tf.group(*model.update_emas)\n merged = tf.summary.merge_all() # summary\n train_writer = tf.summary.FileWriter(summary_path + 'train', sess.graph)\n test_writer = tf.summary.FileWriter(summary_path + 'test')\n training_ops = [v for v in tf.global_variables() if v.name.startswith(vs.name+'/')]\n\n # 如果已经保存过模型，导入上次的模型\n if os.path.exists(ckpt_path + \"checkpoint\"):\n print(\"Restoring Variables from Checkpoint...\")\n model.saver.restore(sess, tf.train.latest_checkpoint(ckpt_path))\n last_valid_cost, precision, recall, last_f1 = valid_epoch(data_valid_path, sess, model)\n print(' valid cost=%g; p=%g, r=%g, f1=%g' % (last_valid_cost, precision, recall, last_f1))\n sess.run(tf.variables_initializer(training_ops))\n train_op2 = train_op1\n else:\n print('Initializing Variables...')\n sess.run(tf.global_variables_initializer())\n\n print('3.Begin training...')\n print('max_epoch=%d, max_max_epoch=%d' % (FLAGS.max_epoch, FLAGS.max_max_epoch))\n train_op = train_op2\n for epoch in range(FLAGS.max_max_epoch):\n global_step = sess.run(model.global_step)\n print('Global step %d, lr=%g' % (global_step, sess.run(learning_rate)))\n if epoch == FLAGS.max_epoch: # update the embedding\n train_op = train_op1\n train_fetches = [merged, model.loss, train_op, update_op]\n valid_fetches = [merged, model.loss]\n train_epoch(data_train_path, sess, model, train_fetches, valid_fetches, train_writer, test_writer)\n # 最后再做一次验证\n valid_cost, precision, recall, f1 = valid_epoch(data_valid_path, sess, model)\n print('END.Global_step=%d: valid cost=%g; p=%g, r=%g, f1=%g' % (\n sess.run(model.global_step), valid_cost, precision, recall, f1))\n if f1 > last_f1: # save the better model\n saving_path = model.saver.save(sess, model_path, sess.run(model.global_step)+1)\n print('saved new model to %s ' % saving_path)\n\n\nif __name__ == '__main__':\n tf.app.run()\n" }, { "path": "zhihu-text-classification-master/models/wd_3_bigru/__init__.py", "content": "# -*- coding:utf-8 -*- \n\n" }, { "path": "zhihu-text-classification-master/models/wd_3_bigru/network.py", "content": "# -*- coding:utf-8 -*-\n\nimport tensorflow as tf\nfrom tensorflow.contrib import rnn\nimport tensorflow.contrib.layers as layers\n\n\"\"\"wd_3_bigru\ntitle 部分使用 bigru+attention；content 部分使用 bigru+attention； 两部分输出直接 concat。\n\"\"\"\n\n\nclass Settings(object):\n def __init__(self):\n self.model_name = 'wd_3_bigru'\n self.title_len = 30\n self.content_len = 150\n self.hidden_size = 256\n self.n_layer = 1\n self.fc_hidden_size = 1024\n self.n_class = 1999\n self.summary_path = '../../summary/' + self.model_name + '/'\n self.ckpt_path = '../../ckpt/' + self.model_name + '/'\n\n\nclass BiGRU(object):\n \"\"\"\n title: inputs->bigru+attention->output_title\n content: inputs->bigru+attention->output_content\n concat[output_title, output_content] -> fc+bn+relu -> sigmoid_entropy.\n \"\"\"\n\n def __init__(self, W_embedding, settings):\n self.model_name = settings.model_name\n self.title_len = settings.title_len\n self.content_len = settings.content_len\n self.hidden_size = settings.hidden_size\n self.n_layer = settings.n_layer\n self.n_class = settings.n_class\n self.fc_hidden_size = settings.fc_hidden_size\n self._global_step = tf.Variable(0, trainable=False, name='Global_Step')\n self.update_emas = list()\n # placeholders\n self._tst = tf.placeholder(tf.bool)\n self._keep_prob = tf.placeholder(tf.float32, [])\n self._batch_size = tf.placeholder(tf.int32, [])\n\n with tf.name_scope('Inputs'):\n self._X1_inputs = tf.placeholder(tf.int64, [None, self.title_len], name='X1_inputs')\n self._X2_inputs = tf.placeholder(tf.int64, [None, self.content_len], name='X2_inputs')\n self._y_inputs = tf.placeholder(tf.float32, [None, self.n_class], name='y_input')\n\n with tf.variable_scope('embedding'):\n self.embedding = tf.get_variable(name='embedding', shape=W_embedding.shape,\n initializer=tf.constant_initializer(W_embedding), trainable=True)\n self.embedding_size = W_embedding.shape[1]\n\n with tf.variable_scope('bigru_text'):\n output_title = self.bigru_inference(self._X1_inputs)\n\n with tf.variable_scope('bigru_content'):\n output_content = self.bigru_inference(self._X2_inputs)\n\n with tf.variable_scope('fc-bn-layer'):\n output = tf.concat([output_title, output_content], axis=1)\n W_fc = self.weight_variable([self.hidden_size * 4, self.fc_hidden_size], name='Weight_fc')\n tf.summary.histogram('W_fc', W_fc)\n h_fc = tf.matmul(output, W_fc, name='h_fc')\n beta_fc = tf.Variable(tf.constant(0.1, tf.float32, shape=[self.fc_hidden_size], name=\"beta_fc\"))\n tf.summary.histogram('beta_fc', beta_fc)\n fc_bn, update_ema_fc = self.batchnorm(h_fc, beta_fc, convolutional=False)\n self.update_emas.append(update_ema_fc)\n self.fc_bn_relu = tf.nn.relu(fc_bn, name=\"relu\")\n\n with tf.variable_scope('out_layer'):\n W_out = self.weight_variable([self.fc_hidden_size, self.n_class], name='Weight_out')\n tf.summary.histogram('Weight_out', W_out)\n b_out = self.bias_variable([self.n_class], name='bias_out')\n tf.summary.histogram('bias_out', b_out)\n self._y_pred = tf.nn.xw_plus_b(self.fc_bn_relu, W_out, b_out, name='y_pred') # 每个类别的分数 scores\n\n with tf.name_scope('loss'):\n self._loss = tf.reduce_mean(\n tf.nn.sigmoid_cross_entropy_with_logits(logits=self._y_pred, labels=self._y_inputs))\n tf.summary.scalar('loss', self._loss)\n\n self.saver = tf.train.Saver(max_to_keep=1)\n\n @property\n def tst(self):\n return self._tst\n\n @property\n def keep_prob(self):\n return self._keep_prob\n\n @property\n def batch_size(self):\n return self._batch_size\n\n @property\n def global_step(self):\n return self._global_step\n\n @property\n def X1_inputs(self):\n return self._X1_inputs\n\n @property\n def X2_inputs(self):\n return self._X2_inputs\n\n @property\n def y_inputs(self):\n return self._y_inputs\n\n @property\n def y_pred(self):\n return self._y_pred\n\n @property\n def loss(self):\n return self._loss\n\n def weight_variable(self, shape, name):\n \"\"\"Create a weight variable with appropriate initialization.\"\"\"\n initial = tf.truncated_normal(shape, stddev=0.1)\n return tf.Variable(initial, name=name)\n\n def bias_variable(self, shape, name):\n \"\"\"Create a bias variable with appropriate initialization.\"\"\"\n initial = tf.constant(0.1, shape=shape)\n return tf.Variable(initial, name=name)\n\n def batchnorm(self, Ylogits, offset, convolutional=False):\n \"\"\"batchnormalization.\n Args:\n Ylogits: 1D向量或者是3D的卷积结果。\n num_updates: 迭代的global_step\n offset：表示beta，全局均值；在 RELU 激活中一般初始化为 0.1。\n scale：表示lambda，全局方差；在 sigmoid 激活中需要，这 RELU 激活中作用不大。\n m: 表示batch均值；v:表示batch方差。\n bnepsilon：一个很小的浮点数，防止除以 0.\n Returns:\n Ybn: 和 Ylogits 的维度一样，就是经过 Batch Normalization 处理的结果。\n update_moving_everages：更新mean和variance，主要是给最后的 test 使用。\n \"\"\"\n exp_moving_avg = tf.train.ExponentialMovingAverage(0.999, self._global_step) # adding the iteration prevents from averaging across non-existing iterations\n bnepsilon = 1e-5\n if convolutional:\n mean, variance = tf.nn.moments(Ylogits, [0, 1, 2])\n else:\n mean, variance = tf.nn.moments(Ylogits, [0])\n update_moving_everages = exp_moving_avg.apply([mean, variance])\n m = tf.cond(self.tst, lambda: exp_moving_avg.average(mean), lambda: mean)\n v = tf.cond(self.tst, lambda: exp_moving_avg.average(variance), lambda: variance)\n Ybn = tf.nn.batch_normalization(Ylogits, m, v, offset, None, bnepsilon)\n return Ybn, update_moving_everages\n\n def gru_cell(self):\n with tf.name_scope('gru_cell'):\n cell = rnn.GRUCell(self.hidden_size, reuse=tf.get_variable_scope().reuse)\n return rnn.DropoutWrapper(cell, output_keep_prob=self.keep_prob)\n\n def bi_gru(self, inputs):\n \"\"\"build the bi-GRU network. 返回个所有层的隐含状态。\"\"\"\n cells_fw = [self.gru_cell() for _ in range(self.n_layer)]\n cells_bw = [self.gru_cell() for _ in range(self.n_layer)]\n initial_states_fw = [cell_fw.zero_state(self.batch_size, tf.float32) for cell_fw in cells_fw]\n initial_states_bw = [cell_bw.zero_state(self.batch_size, tf.float32) for cell_bw in cells_bw]\n outputs, _, _ = rnn.stack_bidirectional_dynamic_rnn(cells_fw, cells_bw, inputs,\n initial_states_fw=initial_states_fw,\n initial_states_bw=initial_states_bw, dtype=tf.float32)\n return outputs\n\n def task_specific_attention(self, inputs, output_size,\n initializer=layers.xavier_initializer(),\n activation_fn=tf.tanh, scope=None):\n \"\"\"\n Performs task-specific attention reduction, using learned\n attention context vector (constant within task of interest).\n Args:\n inputs: Tensor of shape [batch_size, units, input_size]\n `input_size` must be static (known)\n `units` axis will be attended over (reduced from output)\n `batch_size` will be preserved\n output_size: Size of output's inner (feature) dimension\n Returns:\n outputs: Tensor of shape [batch_size, output_dim].\n \"\"\"\n assert len(inputs.get_shape()) == 3 and inputs.get_shape()[-1].value is not None\n with tf.variable_scope(scope or 'attention') as scope:\n # u_w, attention 向量\n attention_context_vector = tf.get_variable(name='attention_context_vector', shape=[output_size],\n initializer=initializer, dtype=tf.float32)\n # 全连接层，把 h_i 转为 u_i ， shape= [batch_size, units, input_size] -> [batch_size, units, output_size]\n input_projection = layers.fully_connected(inputs, output_size, activation_fn=activation_fn, scope=scope)\n # 输出 [batch_size, units]\n vector_attn = tf.reduce_sum(tf.multiply(input_projection, attention_context_vector), axis=2, keep_dims=True)\n attention_weights = tf.nn.softmax(vector_attn, dim=1)\n tf.summary.histogram('attention_weigths', attention_weights)\n weighted_projection = tf.multiply(inputs, attention_weights)\n outputs = tf.reduce_sum(weighted_projection, axis=1)\n return outputs # 输出 [batch_size, hidden_size*2]\n\n def bigru_inference(self, X_inputs):\n inputs = tf.nn.embedding_lookup(self.embedding, X_inputs)\n output_bigru = self.bi_gru(inputs)\n output_att = self.task_specific_attention(output_bigru, self.hidden_size*2)\n return output_att\n\n\n# test the model\ndef test():\n import numpy as np\n print('Begin testing...')\n settings = Settings()\n W_embedding = np.random.randn(50, 10)\n config = tf.ConfigProto()\n config.gpu_options.allow_growth = True\n batch_size = 128\n with tf.Session(config=config) as sess:\n model = BiGRU(W_embedding, settings)\n optimizer = tf.train.AdamOptimizer(0.001)\n train_op = optimizer.minimize(model.loss)\n update_op = tf.group(*model.update_emas)\n sess.run(tf.global_variables_initializer())\n fetch = [model.loss, model.y_pred, train_op, update_op]\n loss_list = list()\n for i in xrange(100):\n X1_batch = np.zeros((batch_size, 30), dtype=float)\n X2_batch = np.zeros((batch_size, 150), dtype=float)\n y_batch = np.zeros((batch_size, 1999), dtype=int)\n _batch_size = len(y_batch)\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: False, model.keep_prob: 0.5}\n loss, y_pred, _, _ = sess.run(fetch, feed_dict=feed_dict)\n loss_list.append(loss)\n print(i, loss)\n\nif __name__ == '__main__':\n test()\n" }, { "path": "zhihu-text-classification-master/models/wd_3_bigru/predict.py", "content": "# -*- coding:utf-8 -*-\n\nfrom __future__ import print_function\nfrom __future__ import division\nimport tensorflow as tf\nimport numpy as np\nfrom tqdm import tqdm\nimport os\nimport sys\nimport time\nimport network\n\nsys.path.append('../..')\nfrom evaluator import score_eval\n\nsettings = network.Settings()\ntitle_len = settings.title_len\nmodel_name = settings.model_name\nckpt_path = settings.ckpt_path\n\nlocal_scores_path = '../../local_scores/'\nscores_path = '../../scores/'\nif not os.path.exists(local_scores_path):\n os.makedirs(local_scores_path)\nif not os.path.exists(scores_path):\n os.makedirs(scores_path)\n\nembedding_path = '../../data/word_embedding.npy'\ndata_valid_path = '../../data/wd-data/data_valid/'\ndata_test_path = '../../data/wd-data/data_test/'\nva_batches = os.listdir(data_valid_path)\nte_batches = os.listdir(data_test_path) # batch 文件名列表\nn_va_batches = len(va_batches)\nn_te_batches = len(te_batches)\n\n\ndef get_batch(batch_id):\n \"\"\"get a batch from valid data\"\"\"\n new_batch = np.load(data_valid_path + str(batch_id) + '.npz')\n X_batch = new_batch['X']\n y_batch = new_batch['y']\n X1_batch = X_batch[:, :title_len]\n X2_batch = X_batch[:, title_len:]\n return [X1_batch, X2_batch, y_batch]\n\n\ndef get_test_batch(batch_id):\n \"\"\"get a batch from test data\"\"\"\n X_batch = np.load(data_test_path + str(batch_id) + '.npy')\n X1_batch = X_batch[:, :title_len]\n X2_batch = X_batch[:, title_len:]\n return [X1_batch, X2_batch]\n\n\ndef local_predict(sess, model):\n \"\"\"Test on the valid data.\"\"\"\n time0 = time.time()\n predict_labels_list = list() # 所有的预测结果\n marked_labels_list = list()\n predict_scores = list()\n for i in tqdm(range(n_va_batches)):\n [X1_batch, X2_batch, y_batch] = get_batch(i)\n marked_labels_list.extend(y_batch)\n _batch_size = len(X1_batch)\n fetches = [model.y_pred]\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n predict_labels = sess.run(fetches, feed_dict)[0]\n predict_scores.append(predict_labels)\n predict_labels = list(map(lambda label: label.argsort()[-1:-6:-1], predict_labels)) # 取最大的5个下标\n predict_labels_list.extend(predict_labels)\n predict_label_and_marked_label_list = zip(predict_labels_list, marked_labels_list)\n precision, recall, f1 = score_eval(predict_label_and_marked_label_list)\n print('Local valid p=%g, r=%g, f1=%g' % (precision, recall, f1))\n predict_scores = np.vstack(np.asarray(predict_scores))\n local_scores_name = local_scores_path + model_name + '.npy'\n np.save(local_scores_name, predict_scores)\n print('local_scores.shape=', predict_scores.shape)\n print('Writed the scores into %s, time %g s' % (local_scores_name, time.time() - time0))\n\n\ndef predict(sess, model):\n \"\"\"Test on the test data.\"\"\"\n time0 = time.time()\n predict_scores = list()\n for i in tqdm(range(n_te_batches)):\n [X1_batch, X2_batch] = get_test_batch(i)\n _batch_size = len(X1_batch)\n fetches = [model.y_pred]\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n predict_labels = sess.run(fetches, feed_dict)[0]\n predict_scores.append(predict_labels)\n predict_scores = np.vstack(np.asarray(predict_scores))\n scores_name = scores_path + model_name + '.npy'\n np.save(scores_name, predict_scores)\n print('scores.shape=', predict_scores.shape)\n print('Writed the scores into %s, time %g s' % (scores_name, time.time() - time0))\n\n\ndef main(_):\n if not os.path.exists(ckpt_path + 'checkpoint'):\n print('there is not saved model, please check the ckpt path')\n exit()\n print('Loading model...')\n W_embedding = np.load(embedding_path)\n config = tf.ConfigProto()\n config.gpu_options.allow_growth = True\n with tf.Session(config=config) as sess:\n model = network.BiGRU(W_embedding, settings)\n model.saver.restore(sess, tf.train.latest_checkpoint(ckpt_path))\n print('Local predicting...')\n local_predict(sess, model)\n print('Test predicting...')\n predict(sess, model)\n\n\nif __name__ == '__main__':\n tf.app.run()\n" }, { "path": "zhihu-text-classification-master/models/wd_3_bigru/train.py", "content": "# -*- coding:utf-8 -*-\n\nfrom __future__ import print_function\nfrom __future__ import division\nimport tensorflow as tf\nimport numpy as np\nfrom tqdm import tqdm\nimport os\nimport sys\nimport shutil\nimport time\nimport network\n\nsys.path.append('../..')\nfrom data_helpers import to_categorical\nfrom evaluator import score_eval\n\nflags = tf.flags\nflags.DEFINE_bool('is_retrain', False, 'if is_retrain is true, not rebuild the summary')\nflags.DEFINE_integer('max_epoch', 1, 'update the embedding after max_epoch, default: 1')\nflags.DEFINE_integer('max_max_epoch', 6, 'all training epoches, default: 6')\nflags.DEFINE_float('lr', 8e-4, 'initial learning rate, default: 8e-4')\nflags.DEFINE_float('decay_rate', 0.85, 'decay rate, default: 0.85')\nflags.DEFINE_float('keep_prob', 0.5, 'keep_prob for training, default: 0.5')\n# 正式\nflags.DEFINE_integer('decay_step', 15000, 'decay_step, default: 15000')\nflags.DEFINE_integer('valid_step', 10000, 'valid_step, default: 10000')\nflags.DEFINE_float('last_f1', 0.40, 'if valid_f1 > last_f1, save new model. default: 0.40')\n\n# 测试\n# flags.DEFINE_integer('decay_step', 1000, 'decay_step, default: 1000')\n# flags.DEFINE_integer('valid_step', 500, 'valid_step, default: 500')\n# flags.DEFINE_float('last_f1', 0.10, 'if valid_f1 > last_f1, save new model. default: 0.10')\nFLAGS = flags.FLAGS\n\nlr = FLAGS.lr\nlast_f1 = FLAGS.last_f1\nsettings = network.Settings()\ntitle_len = settings.title_len\nsummary_path = settings.summary_path\nckpt_path = settings.ckpt_path\nmodel_path = ckpt_path + 'model.ckpt'\n\nembedding_path = '../../data/word_embedding.npy'\ndata_train_path = '../../data/wd-data/data_train/'\ndata_valid_path = '../../data/wd-data/data_valid/'\ntr_batches = os.listdir(data_train_path) # batch 文件名列表\nva_batches = os.listdir(data_valid_path)\nn_tr_batches = len(tr_batches)\nn_va_batches = len(va_batches)\n\n# 测试\n# n_tr_batches = 1000\n# n_va_batches = 50\n\n\ndef get_batch(data_path, batch_id):\n \"\"\"get a batch from data_path\"\"\"\n new_batch = np.load(data_path + str(batch_id) + '.npz')\n X_batch = new_batch['X']\n y_batch = new_batch['y']\n X1_batch = X_batch[:, :title_len]\n X2_batch = X_batch[:, title_len:]\n return [X1_batch, X2_batch, y_batch]\n\n\ndef valid_epoch(data_path, sess, model):\n \"\"\"Test on the valid data.\"\"\"\n va_batches = os.listdir(data_path)\n n_va_batches = len(va_batches)\n _costs = 0.0\n predict_labels_list = list() # 所有的预测结果\n marked_labels_list = list()\n for i in range(n_va_batches):\n [X1_batch, X2_batch, y_batch] = get_batch(data_path, i)\n marked_labels_list.extend(y_batch)\n y_batch = to_categorical(y_batch)\n _batch_size = len(y_batch)\n fetches = [model.loss, model.y_pred]\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n _cost, predict_labels = sess.run(fetches, feed_dict)\n _costs += _cost\n predict_labels = list(map(lambda label: label.argsort()[-1:-6:-1], predict_labels)) # 取最大的5个下标\n predict_labels_list.extend(predict_labels)\n predict_label_and_marked_label_list = zip(predict_labels_list, marked_labels_list)\n precision, recall, f1 = score_eval(predict_label_and_marked_label_list)\n mean_cost = _costs / n_va_batches\n return mean_cost, precision, recall, f1\n\n\ndef train_epoch(data_path, sess, model, train_fetches, valid_fetches, train_writer, test_writer):\n global last_f1\n global lr\n time0 = time.time()\n batch_indexs = np.random.permutation(n_tr_batches) # shuffle the training data\n for batch in tqdm(range(n_tr_batches)):\n global_step = sess.run(model.global_step)\n if 0 == (global_step + 1) % FLAGS.valid_step:\n valid_cost, precision, recall, f1 = valid_epoch(data_valid_path, sess, model)\n print('Global_step=%d: valid cost=%g; p=%g, r=%g, f1=%g, time=%g s' % (\n global_step, valid_cost, precision, recall, f1, time.time() - time0))\n time0 = time.time()\n if f1 > last_f1:\n last_f1 = f1\n saving_path = model.saver.save(sess, model_path, global_step+1)\n print('saved new model to %s ' % saving_path)\n # training\n batch_id = batch_indexs[batch]\n [X1_batch, X2_batch, y_batch] = get_batch(data_train_path, batch_id)\n y_batch = to_categorical(y_batch)\n _batch_size = len(y_batch)\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: False, model.keep_prob: FLAGS.keep_prob}\n summary, _cost, _, _ = sess.run(train_fetches, feed_dict) # the cost is the mean cost of one batch\n # valid per 500 steps\n if 0 == (global_step + 1) % 500:\n train_writer.add_summary(summary, global_step)\n batch_id = np.random.randint(0, n_va_batches) # 随机选一个验证batch\n [X1_batch, X2_batch, y_batch] = get_batch(data_valid_path, batch_id)\n y_batch = to_categorical(y_batch)\n _batch_size = len(y_batch)\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n summary, _cost = sess.run(valid_fetches, feed_dict)\n test_writer.add_summary(summary, global_step)\n\n\ndef main(_):\n global ckpt_path\n global last_f1\n if not os.path.exists(ckpt_path):\n os.makedirs(ckpt_path)\n if not os.path.exists(summary_path):\n os.makedirs(summary_path)\n elif not FLAGS.is_retrain: # 重新训练本模型，删除以前的 summary\n shutil.rmtree(summary_path)\n os.makedirs(summary_path)\n if not os.path.exists(summary_path):\n os.makedirs(summary_path)\n\n print('1.Loading data...')\n W_embedding = np.load(embedding_path)\n print('training sample_num = %d' % n_tr_batches)\n print('valid sample_num = %d' % n_va_batches)\n\n # Initial or restore the model\n print('2.Building model...')\n config = tf.ConfigProto()\n config.gpu_options.allow_growth = True\n with tf.Session(config=config) as sess:\n model = network.BiGRU(W_embedding, settings)\n with tf.variable_scope('training_ops') as vs:\n learning_rate = tf.train.exponential_decay(FLAGS.lr, model.global_step, FLAGS.decay_step,\n FLAGS.decay_rate, staircase=True)\n # two optimizer: op1, update embedding; op2, do not update embedding.\n with tf.variable_scope('Optimizer1'):\n tvars1 = tf.trainable_variables()\n grads1 = tf.gradients(model.loss, tvars1)\n optimizer1 = tf.train.AdamOptimizer(learning_rate=learning_rate)\n train_op1 = optimizer1.apply_gradients(zip(grads1, tvars1),\n global_step=model.global_step)\n with tf.variable_scope('Optimizer2'):\n tvars2 = [tvar for tvar in tvars1 if 'embedding' not in tvar.name]\n grads2 = tf.gradients(model.loss, tvars2)\n optimizer2 = tf.train.AdamOptimizer(learning_rate=learning_rate)\n train_op2 = optimizer2.apply_gradients(zip(grads2, tvars2),\n global_step=model.global_step)\n update_op = tf.group(*model.update_emas)\n merged = tf.summary.merge_all() # summary\n train_writer = tf.summary.FileWriter(summary_path + 'train', sess.graph)\n test_writer = tf.summary.FileWriter(summary_path + 'test')\n training_ops = [v for v in tf.global_variables() if v.name.startswith(vs.name+'/')]\n\n # 如果已经保存过模型，导入上次的模型\n if os.path.exists(ckpt_path + \"checkpoint\"):\n print(\"Restoring Variables from Checkpoint...\")\n model.saver.restore(sess, tf.train.latest_checkpoint(ckpt_path))\n last_valid_cost, precision, recall, last_f1 = valid_epoch(data_valid_path, sess, model)\n print(' valid cost=%g; p=%g, r=%g, f1=%g' % (last_valid_cost, precision, recall, last_f1))\n sess.run(tf.variables_initializer(training_ops))\n train_op2 = train_op1\n else:\n print('Initializing Variables...')\n sess.run(tf.global_variables_initializer())\n\n print('3.Begin training...')\n\n train_op = train_op2\n print('max_epoch=%d, max_max_epoch=%d' % (FLAGS.max_epoch, FLAGS.max_max_epoch))\n for epoch in range(FLAGS.max_max_epoch):\n global_step = sess.run(model.global_step)\n print('Global step %d, lr=%g' % (global_step, sess.run(learning_rate)))\n if epoch == FLAGS.max_epoch: # update the embedding\n train_op = train_op1\n train_fetches = [merged, model.loss, train_op, update_op]\n valid_fetches = [merged, model.loss]\n train_epoch(data_train_path, sess, model, train_fetches, valid_fetches, train_writer, test_writer)\n # 最后再做一次验证\n valid_cost, precision, recall, f1 = valid_epoch(data_valid_path, sess, model)\n print('END.Global_step=%d: valid cost=%g; p=%g, r=%g, f1=%g' % (\n sess.run(model.global_step), valid_cost, precision, recall, f1))\n if f1 > last_f1: # save the better model\n saving_path = model.saver.save(sess, model_path, sess.run(model.global_step)+1)\n print('saved new model to %s ' % saving_path)\n\n\nif __name__ == '__main__':\n tf.app.run()\n" }, { "path": "zhihu-text-classification-master/models/wd_4_han/__init__.py", "content": "# -*- coding:utf-8 -*- \n\n" }, { "path": "zhihu-text-classification-master/models/wd_4_han/network.py", "content": "# -*- coding:utf-8 -*-\n\nimport tensorflow as tf\nfrom tensorflow.contrib import rnn\nimport tensorflow.contrib.layers as layers\n\n\"\"\"wd_4_han\ntitle 部分使用 bigru+attention；content 部分使用 han； 两部分输出直接 concat。\n\"\"\"\n\n\nclass Settings(object):\n def __init__(self):\n self.model_name = 'wd_4_han'\n self.title_len = self.sent_len = 30\n self.doc_len = 10\n self.hidden_size = 256\n self.n_layer = 1\n self.fc_hidden_size = 1024\n self.n_class = 1999\n self.summary_path = '../../summary/' + self.model_name + '/'\n self.ckpt_path = '../../ckpt/' + self.model_name + '/'\n\n\nclass HAN(object):\n \"\"\"\n title: inputs->bigru+attention->output_title\n content: inputs->sent_encoder(bigru+attention)->doc_encoder(bigru+attention)->output_content\n concat[output_title, output_content] -> fc+bn+relu -> sigmoid_entropy.\n \"\"\"\n\n def __init__(self, W_embedding, settings):\n self.model_name = settings.model_name\n self.title_len = self.sent_len = settings.sent_len\n self.doc_len = settings.doc_len\n self.hidden_size = settings.hidden_size\n self.n_layer = settings.n_layer\n self.n_class = settings.n_class\n self.fc_hidden_size = settings.fc_hidden_size\n self._global_step = tf.Variable(0, trainable=False, name='Global_Step')\n self.update_emas = list()\n # placeholders\n self._tst = tf.placeholder(tf.bool)\n self._keep_prob = tf.placeholder(tf.float32, [])\n self._batch_size = tf.placeholder(tf.int32, [])\n\n with tf.name_scope('Inputs'):\n self._X1_inputs = tf.placeholder(tf.int64, [None, self.title_len], name='X1_inputs')\n self._X2_inputs = tf.placeholder(tf.int64, [None, self.doc_len * self.sent_len], name='X2_inputs')\n self._y_inputs = tf.placeholder(tf.float32, [None, self.n_class], name='y_input')\n\n with tf.variable_scope('embedding'):\n self.embedding = tf.get_variable(name='embedding', shape=W_embedding.shape,\n initializer=tf.constant_initializer(W_embedding), trainable=True)\n self.embedding_size = W_embedding.shape[1]\n\n with tf.variable_scope('bigru_text'):\n output_title = self.bigru_inference(self._X1_inputs)\n\n with tf.variable_scope('han_content'):\n output_content = self.han_inference(self._X2_inputs)\n\n with tf.variable_scope('fc-bn-layer'):\n output = tf.concat([output_title, output_content], axis=1)\n W_fc = self.weight_variable([self.hidden_size * 4, self.fc_hidden_size], name='Weight_fc')\n tf.summary.histogram('W_fc', W_fc)\n h_fc = tf.matmul(output, W_fc, name='h_fc')\n beta_fc = tf.Variable(tf.constant(0.1, tf.float32, shape=[self.fc_hidden_size], name=\"beta_fc\"))\n tf.summary.histogram('beta_fc', beta_fc)\n fc_bn, update_ema_fc = self.batchnorm(h_fc, beta_fc, convolutional=False)\n self.update_emas.append(update_ema_fc)\n self.fc_bn_relu = tf.nn.relu(fc_bn, name=\"relu\")\n fc_bn_drop = tf.nn.dropout(self.fc_bn_relu, self.keep_prob)\n\n with tf.variable_scope('out_layer'):\n W_out = self.weight_variable([self.fc_hidden_size, self.n_class], name='Weight_out')\n tf.summary.histogram('Weight_out', W_out)\n b_out = self.bias_variable([self.n_class], name='bias_out')\n tf.summary.histogram('bias_out', b_out)\n self._y_pred = tf.nn.xw_plus_b(fc_bn_drop, W_out, b_out, name='y_pred') # 每个类别的分数 scores\n\n with tf.name_scope('loss'):\n self._loss = tf.reduce_mean(\n tf.nn.sigmoid_cross_entropy_with_logits(logits=self._y_pred, labels=self._y_inputs))\n tf.summary.scalar('loss', self._loss)\n\n self.saver = tf.train.Saver(max_to_keep=1)\n\n @property\n def tst(self):\n return self._tst\n\n @property\n def keep_prob(self):\n return self._keep_prob\n\n @property\n def batch_size(self):\n return self._batch_size\n\n @property\n def global_step(self):\n return self._global_step\n\n @property\n def X1_inputs(self):\n return self._X1_inputs\n\n @property\n def X2_inputs(self):\n return self._X2_inputs\n\n @property\n def y_inputs(self):\n return self._y_inputs\n\n @property\n def y_pred(self):\n return self._y_pred\n\n @property\n def loss(self):\n return self._loss\n\n def weight_variable(self, shape, name):\n \"\"\"Create a weight variable with appropriate initialization.\"\"\"\n initial = tf.truncated_normal(shape, stddev=0.1)\n return tf.Variable(initial, name=name)\n\n def bias_variable(self, shape, name):\n \"\"\"Create a bias variable with appropriate initialization.\"\"\"\n initial = tf.constant(0.1, shape=shape)\n return tf.Variable(initial, name=name)\n\n def batchnorm(self, Ylogits, offset, convolutional=False):\n \"\"\"batchnormalization.\n Args:\n Ylogits: 1D向量或者是3D的卷积结果。\n num_updates: 迭代的global_step\n offset：表示beta，全局均值；在 RELU 激活中一般初始化为 0.1。\n scale：表示lambda，全局方差；在 sigmoid 激活中需要，这 RELU 激活中作用不大。\n m: 表示batch均值；v:表示batch方差。\n bnepsilon：一个很小的浮点数，防止除以 0.\n Returns:\n Ybn: 和 Ylogits 的维度一样，就是经过 Batch Normalization 处理的结果。\n update_moving_everages：更新mean和variance，主要是给最后的 test 使用。\n \"\"\"\n exp_moving_avg = tf.train.ExponentialMovingAverage(0.999, self._global_step) # adding the iteration prevents from averaging across non-existing iterations\n bnepsilon = 1e-5\n if convolutional:\n mean, variance = tf.nn.moments(Ylogits, [0, 1, 2])\n else:\n mean, variance = tf.nn.moments(Ylogits, [0])\n update_moving_everages = exp_moving_avg.apply([mean, variance])\n m = tf.cond(self.tst, lambda: exp_moving_avg.average(mean), lambda: mean)\n v = tf.cond(self.tst, lambda: exp_moving_avg.average(variance), lambda: variance)\n Ybn = tf.nn.batch_normalization(Ylogits, m, v, offset, None, bnepsilon)\n return Ybn, update_moving_everages\n\n def gru_cell(self):\n with tf.name_scope('gru_cell'):\n cell = rnn.GRUCell(self.hidden_size, reuse=tf.get_variable_scope().reuse)\n return rnn.DropoutWrapper(cell, output_keep_prob=self.keep_prob)\n\n def bi_gru(self, inputs, seg_num):\n \"\"\"build the bi-GRU network. Return the encoder represented vector.\n n_step: 句子的词数量；或者文档的句子数。\n seg_num: 序列的数量，原本应该为 batch_size, 但是这里将 batch_size 个 doc展开成很多个句子。\n \"\"\"\n cells_fw = [self.gru_cell() for _ in range(self.n_layer)]\n cells_bw = [self.gru_cell() for _ in range(self.n_layer)]\n initial_states_fw = [cell_fw.zero_state(seg_num, tf.float32) for cell_fw in cells_fw]\n initial_states_bw = [cell_bw.zero_state(seg_num, tf.float32) for cell_bw in cells_bw]\n outputs, _, _ = rnn.stack_bidirectional_dynamic_rnn(cells_fw, cells_bw, inputs,\n initial_states_fw = initial_states_fw, initial_states_bw = initial_states_bw, dtype=tf.float32)\n # outputs: Output Tensor shaped: seg_num, max_time, layers_output]，其中layers_output=hidden_size * 2 在这里。\n return outputs\n\n def task_specific_attention(self, inputs, output_size,\n initializer=layers.xavier_initializer(),\n activation_fn=tf.tanh, scope=None):\n \"\"\"\n Performs task-specific attention reduction, using learned\n attention context vector (constant within task of interest).\n Args:\n inputs: Tensor of shape [batch_size, units, input_size]\n `input_size` must be static (known)\n `units` axis will be attended over (reduced from output)\n `batch_size` will be preserved\n output_size: Size of output's inner (feature) dimension\n Returns:\n outputs: Tensor of shape [batch_size, output_dim].\n \"\"\"\n assert len(inputs.get_shape()) == 3 and inputs.get_shape()[-1].value is not None\n with tf.variable_scope(scope or 'attention') as scope:\n # u_w, attention 向量\n attention_context_vector = tf.get_variable(name='attention_context_vector', shape=[output_size],\n initializer=initializer, dtype=tf.float32)\n # 全连接层，把 h_i 转为 u_i ， shape= [batch_size, units, input_size] -> [batch_size, units, output_size]\n input_projection = layers.fully_connected(inputs, output_size, activation_fn=activation_fn, scope=scope)\n # 输出 [batch_size, units]\n vector_attn = tf.reduce_sum(tf.multiply(input_projection, attention_context_vector), axis=2, keep_dims=True)\n attention_weights = tf.nn.softmax(vector_attn, dim=1)\n tf.summary.histogram('attention_weigths', attention_weights)\n weighted_projection = tf.multiply(inputs, attention_weights)\n outputs = tf.reduce_sum(weighted_projection, axis=1)\n return outputs # 输出 [batch_size, hidden_size*2]\n\n def bigru_inference(self, X_inputs):\n inputs = tf.nn.embedding_lookup(self.embedding, X_inputs)\n output_bigru = self.bi_gru(inputs, self.batch_size)\n output_att = self.task_specific_attention(output_bigru, self.hidden_size*2)\n return output_att # 输出 [batch_size, hidden_size*2]\n\n def han_inference(self, X_inputs):\n \"\"\"分层 attention 模型。content部分。\n Args:\n X_inputs: tensor.shape=(batch_size, doc_len*sent_len)\n Returns:\n doc_attn_outputs: tensor.shape=(batch_size, hidden_size(*2 for bigru))\n \"\"\"\n inputs = tf.nn.embedding_lookup(self.embedding, X_inputs) # inputs.shape=[batch_size, doc_len*sent_len, embedding_size]\n sent_inputs = tf.reshape(inputs,[self.batch_size*self.doc_len, self.sent_len, self.embedding_size]) # shape=(?, 40, 256)\n with tf.variable_scope('sentence_encoder'): # 生成句向量\n sent_outputs = self.bi_gru(sent_inputs, seg_num=self.batch_size*self.doc_len)\n sent_attn_outputs = self.task_specific_attention(sent_outputs, self.hidden_size*2) # [batch_size*doc_len, hidden_size*2]\n with tf.variable_scope('dropout'):\n sent_attn_outputs = tf.nn.dropout(sent_attn_outputs, self.keep_prob)\n with tf.variable_scope('doc_encoder'): # 生成文档向量\n doc_inputs = tf.reshape(sent_attn_outputs, [self.batch_size, self.doc_len, self.hidden_size*2])\n doc_outputs = self.bi_gru(doc_inputs, self.batch_size) # [batch_size, doc_len, hidden_size*2]\n doc_attn_outputs = self.task_specific_attention(doc_outputs, self.hidden_size*2) # [batch_size, hidden_size*2]\n return doc_attn_outputs # [batch_size, hidden_size*2]\n\n\n\n# test the model\ndef test():\n import numpy as np\n print('Begin testing...')\n settings = Settings()\n W_embedding = np.random.randn(50, 10)\n config = tf.ConfigProto()\n config.gpu_options.allow_growth = True\n batch_size = 128\n with tf.Session(config=config) as sess:\n model = HAN(W_embedding, settings)\n optimizer = tf.train.AdamOptimizer(0.001)\n train_op = optimizer.minimize(model.loss)\n update_op = tf.group(*model.update_emas)\n sess.run(tf.global_variables_initializer())\n fetch = [model.loss, model.y_pred, train_op, update_op]\n loss_list = list()\n for i in xrange(100):\n X1_batch = np.zeros((batch_size, 30), dtype=float)\n X2_batch = np.zeros((batch_size, 10 * 30), dtype=float)\n y_batch = np.zeros((batch_size, 1999), dtype=int)\n _batch_size = len(y_batch)\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: False, model.keep_prob: 0.5}\n loss, y_pred, _, _ = sess.run(fetch, feed_dict=feed_dict)\n loss_list.append(loss)\n print(i, loss)\n\nif __name__ == '__main__':\n test()\n" }, { "path": "zhihu-text-classification-master/models/wd_4_han/predict.py", "content": "# -*- coding:utf-8 -*-\n\nfrom __future__ import print_function\nfrom __future__ import division\nimport tensorflow as tf\nimport numpy as np\nfrom tqdm import tqdm\nimport os\nimport sys\nimport time\nimport network\n\nsys.path.append('../..')\nfrom evaluator import score_eval\n\nsettings = network.Settings()\ntitle_len = settings.title_len\nmodel_name = settings.model_name\nckpt_path = settings.ckpt_path\n\nlocal_scores_path = '../../local_scores/'\nscores_path = '../../scores/'\nif not os.path.exists(local_scores_path):\n os.makedirs(local_scores_path)\nif not os.path.exists(scores_path):\n os.makedirs(scores_path)\n\nembedding_path = '../../data/word_embedding.npy'\ndata_valid_path = '../../data/wd-data/seg_valid/'\ndata_test_path = '../../data/wd-data/seg_test/'\nva_batches = os.listdir(data_valid_path)\nte_batches = os.listdir(data_test_path) # batch 文件名列表\nn_va_batches = len(va_batches)\nn_te_batches = len(te_batches)\n\n\ndef get_batch(batch_id):\n \"\"\"get a batch from valid data\"\"\"\n new_batch = np.load(data_valid_path + str(batch_id) + '.npz')\n X_batch = new_batch['X']\n y_batch = new_batch['y']\n X1_batch = X_batch[:, :title_len]\n X2_batch = X_batch[:, title_len:]\n return [X1_batch, X2_batch, y_batch]\n\n\ndef get_test_batch(batch_id):\n \"\"\"get a batch from test data\"\"\"\n X_batch = np.load(data_test_path + str(batch_id) + '.npy')\n X1_batch = X_batch[:, :title_len]\n X2_batch = X_batch[:, title_len:]\n return [X1_batch, X2_batch]\n\n\ndef local_predict(sess, model):\n \"\"\"Test on the valid data.\"\"\"\n time0 = time.time()\n predict_labels_list = list() # 所有的预测结果\n marked_labels_list = list()\n predict_scores = list()\n for i in tqdm(xrange(n_va_batches)):\n [X1_batch, X2_batch, y_batch] = get_batch(i)\n marked_labels_list.extend(y_batch)\n _batch_size = len(X1_batch)\n fetches = [model.y_pred]\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n predict_labels = sess.run(fetches, feed_dict)[0]\n predict_scores.append(predict_labels)\n predict_labels = list(map(lambda label: label.argsort()[-1:-6:-1], predict_labels)) # 取最大的5个下标\n predict_labels_list.extend(predict_labels)\n predict_label_and_marked_label_list = zip(predict_labels_list, marked_labels_list)\n precision, recall, f1 = score_eval(predict_label_and_marked_label_list)\n print('Local valid p=%g, r=%g, f1=%g' % (precision, recall, f1))\n predict_scores = np.vstack(np.asarray(predict_scores))\n local_scores_name = local_scores_path + model_name + '.npy'\n np.save(local_scores_name, predict_scores)\n print('local_scores.shape=', predict_scores.shape)\n print('Writed the scores into %s, time %g s' % (local_scores_name, time.time() - time0))\n\n\ndef predict(sess, model):\n \"\"\"Test on the test data.\"\"\"\n time0 = time.time()\n predict_scores = list()\n for i in tqdm(xrange(n_te_batches)):\n [X1_batch, X2_batch] = get_test_batch(i)\n _batch_size = len(X1_batch)\n fetches = [model.y_pred]\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n predict_labels = sess.run(fetches, feed_dict)[0]\n predict_scores.append(predict_labels)\n predict_scores = np.vstack(np.asarray(predict_scores))\n scores_name = scores_path + model_name + '.npy'\n np.save(scores_name, predict_scores)\n print('scores.shape=', predict_scores.shape)\n print('Writed the scores into %s, time %g s' % (scores_name, time.time() - time0))\n\n\ndef main(_):\n if not os.path.exists(ckpt_path + 'checkpoint'):\n print('there is not saved model, please check the ckpt path')\n exit()\n print('Loading model...')\n W_embedding = np.load(embedding_path)\n config = tf.ConfigProto()\n config.gpu_options.allow_growth = True\n with tf.Session(config=config) as sess:\n model = network.HAN(W_embedding, settings)\n model.saver.restore(sess, tf.train.latest_checkpoint(ckpt_path))\n print('Local predicting...')\n local_predict(sess, model)\n print('Test predicting...')\n predict(sess, model)\n\n\nif __name__ == '__main__':\n tf.app.run()\n" }, { "path": "zhihu-text-classification-master/models/wd_4_han/train.py", "content": "# -*- coding:utf-8 -*-\n\nfrom __future__ import print_function\nfrom __future__ import division\nimport tensorflow as tf\nimport numpy as np\nfrom tqdm import tqdm\nimport os\nimport sys\nimport shutil\nimport time\nimport network\n\nsys.path.append('../..')\nfrom data_helpers import to_categorical\nfrom evaluator import score_eval\n\nflags = tf.flags\nflags.DEFINE_bool('is_retrain', False, 'if is_retrain is true, not rebuild the summary')\nflags.DEFINE_integer('max_epoch', 2, 'update the embedding after max_epoch, default: 2')\nflags.DEFINE_integer('max_max_epoch', 6, 'all training epoches, default: 6')\nflags.DEFINE_float('lr', 8e-4, 'initial learning rate, default: 8e-4')\nflags.DEFINE_float('decay_rate', 0.85, 'decay rate, default: 0.85')\nflags.DEFINE_float('keep_prob', 0.5, 'keep_prob for training, default: 0.5')\n# 正式\nflags.DEFINE_integer('decay_step', 15000, 'decay_step, default: 15000')\nflags.DEFINE_integer('valid_step', 10000, 'valid_step, default: 10000')\nflags.DEFINE_float('last_f1', 0.38, 'if valid_f1 > last_f1, save new model. default: 0.40')\n\n# 测试\n# flags.DEFINE_integer('decay_step', 1000, 'decay_step, default: 1000')\n# flags.DEFINE_integer('valid_step', 500, 'valid_step, default: 500')\n# flags.DEFINE_float('last_f1', 0.10, 'if valid_f1 > last_f1, save new model. default: 0.10')\nFLAGS = flags.FLAGS\n\nlr = FLAGS.lr\nlast_f1 = FLAGS.last_f1\nsettings = network.Settings()\ntitle_len = settings.title_len\nsummary_path = settings.summary_path\nckpt_path = settings.ckpt_path\nmodel_path = ckpt_path + 'model.ckpt'\n\nembedding_path = '../../data/word_embedding.npy'\ndata_train_path = '../../data/wd-data/seg_train/'\ndata_valid_path = '../../data/wd-data/seg_valid/'\ntr_batches = os.listdir(data_train_path) # batch 文件名列表\nva_batches = os.listdir(data_valid_path)\nn_tr_batches = len(tr_batches)\nn_va_batches = len(va_batches)\n\n# 测试\n# n_tr_batches = 1000\n# n_va_batches = 50\n\n\ndef get_batch(data_path, batch_id):\n \"\"\"get a batch from data_path\"\"\"\n new_batch = np.load(data_path + str(batch_id) + '.npz')\n X_batch = new_batch['X']\n y_batch = new_batch['y']\n X1_batch = X_batch[:, :title_len]\n X2_batch = X_batch[:, title_len:]\n return [X1_batch, X2_batch, y_batch]\n\n\ndef valid_epoch(data_path, sess, model):\n \"\"\"Test on the valid data.\"\"\"\n va_batches = os.listdir(data_path)\n n_va_batches = len(va_batches)\n _costs = 0.0\n predict_labels_list = list() # 所有的预测结果\n marked_labels_list = list()\n for i in range(n_va_batches):\n [X1_batch, X2_batch, y_batch] = get_batch(data_path, i)\n marked_labels_list.extend(y_batch)\n y_batch = to_categorical(y_batch)\n _batch_size = len(y_batch)\n fetches = [model.loss, model.y_pred]\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n _cost, predict_labels = sess.run(fetches, feed_dict)\n _costs += _cost\n predict_labels = (map(lambda label: label.argsort()[-1:-6:-1], predict_labels)) # 取最大的5个下标\n predict_labels_list.extend(predict_labels)\n predict_label_and_marked_label_list = zip(predict_labels_list, marked_labels_list)\n precision, recall, f1 = score_eval(predict_label_and_marked_label_list)\n mean_cost = _costs / n_va_batches\n return mean_cost, precision, recall, f1\n\n\ndef train_epoch(data_path, sess, model, train_fetches, valid_fetches, train_writer, test_writer):\n global last_f1\n global lr\n time0 = time.time()\n batch_indexs = np.random.permutation(n_tr_batches) # shuffle the training data\n for batch in tqdm(range(n_tr_batches)):\n global_step = sess.run(model.global_step)\n if 0 == (global_step + 1) % FLAGS.valid_step:\n valid_cost, precision, recall, f1 = valid_epoch(data_valid_path, sess, model)\n print('Global_step=%d: valid cost=%g; p=%g, r=%g, f1=%g, time=%g s' % (\n global_step, valid_cost, precision, recall, f1, time.time() - time0))\n time0 = time.time()\n if f1 > last_f1:\n last_f1 = f1\n saving_path = model.saver.save(sess, model_path, global_step+1)\n print('saved new model to %s ' % saving_path)\n # training\n batch_id = batch_indexs[batch]\n [X1_batch, X2_batch, y_batch] = get_batch(data_train_path, batch_id)\n y_batch = to_categorical(y_batch)\n _batch_size = len(y_batch)\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: False, model.keep_prob: FLAGS.keep_prob}\n summary, _cost, _, _ = sess.run(train_fetches, feed_dict) # the cost is the mean cost of one batch\n # valid per 500 steps\n if 0 == (global_step + 1) % 500:\n train_writer.add_summary(summary, global_step)\n batch_id = np.random.randint(0, n_va_batches) # 随机选一个验证batch\n [X1_batch, X2_batch, y_batch] = get_batch(data_valid_path, batch_id)\n y_batch = to_categorical(y_batch)\n _batch_size = len(y_batch)\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n summary, _cost = sess.run(valid_fetches, feed_dict)\n test_writer.add_summary(summary, global_step)\n\n\ndef main(_):\n global ckpt_path\n global last_f1\n if not os.path.exists(ckpt_path):\n os.makedirs(ckpt_path)\n if not os.path.exists(summary_path):\n os.makedirs(summary_path)\n elif not FLAGS.is_retrain: # 重新训练本模型，删除以前的 summary\n shutil.rmtree(summary_path)\n os.makedirs(summary_path)\n if not os.path.exists(summary_path):\n os.makedirs(summary_path)\n\n print('1.Loading data...')\n W_embedding = np.load(embedding_path)\n print('training sample_num = %d' % n_tr_batches)\n print('valid sample_num = %d' % n_va_batches)\n\n # Initial or restore the model\n print('2.Building model...')\n config = tf.ConfigProto()\n config.gpu_options.allow_growth = True\n with tf.Session(config=config) as sess:\n model = network.HAN(W_embedding, settings)\n with tf.variable_scope('training_ops') as vs:\n learning_rate = tf.train.exponential_decay(FLAGS.lr, model.global_step, FLAGS.decay_step,\n FLAGS.decay_rate, staircase=True)\n # two optimizer: op1, update embedding; op2, do not update embedding.\n with tf.variable_scope('Optimizer1'):\n tvars1 = tf.trainable_variables()\n grads1 = tf.gradients(model.loss, tvars1)\n optimizer1 = tf.train.AdamOptimizer(learning_rate=learning_rate)\n train_op1 = optimizer1.apply_gradients(zip(grads1, tvars1),\n global_step=model.global_step)\n with tf.variable_scope('Optimizer2'):\n tvars2 = [tvar for tvar in tvars1 if 'embedding' not in tvar.name]\n grads2 = tf.gradients(model.loss, tvars2)\n optimizer2 = tf.train.AdamOptimizer(learning_rate=learning_rate)\n train_op2 = optimizer2.apply_gradients(zip(grads2, tvars2),\n global_step=model.global_step)\n update_op = tf.group(*model.update_emas)\n merged = tf.summary.merge_all() # summary\n train_writer = tf.summary.FileWriter(summary_path + 'train', sess.graph)\n test_writer = tf.summary.FileWriter(summary_path + 'test')\n training_ops = [v for v in tf.global_variables() if v.name.startswith(vs.name+'/')]\n\n # 如果已经保存过模型，导入上次的模型\n if os.path.exists(ckpt_path + \"checkpoint\"):\n print(\"Restoring Variables from Checkpoint...\")\n model.saver.restore(sess, tf.train.latest_checkpoint(ckpt_path))\n last_valid_cost, precision, recall, last_f1 = valid_epoch(data_valid_path, sess, model)\n print(' valid cost=%g; p=%g, r=%g, f1=%g' % (last_valid_cost, precision, recall, last_f1))\n sess.run(tf.variables_initializer(training_ops))\n train_op2 = train_op1\n else:\n print('Initializing Variables...')\n sess.run(tf.global_variables_initializer())\n\n print('3.Begin training...')\n print('max_epoch=%d, max_max_epoch=%d' % (FLAGS.max_epoch, FLAGS.max_max_epoch))\n train_op = train_op2\n for epoch in range(FLAGS.max_max_epoch):\n global_step = sess.run(model.global_step)\n print('Global step %d, lr=%g' % (global_step, sess.run(learning_rate)))\n if epoch == FLAGS.max_epoch: # update the embedding\n train_op = train_op1\n train_fetches = [merged, model.loss, train_op, update_op]\n valid_fetches = [merged, model.loss]\n train_epoch(data_train_path, sess, model, train_fetches, valid_fetches, train_writer, test_writer)\n # 最后再做一次验证\n valid_cost, precision, recall, f1 = valid_epoch(data_valid_path, sess, model)\n print('END.Global_step=%d: valid cost=%g; p=%g, r=%g, f1=%g' % (\n sess.run(model.global_step), valid_cost, precision, recall, f1))\n if f1 > last_f1: # save the better model\n saving_path = model.saver.save(sess, model_path, sess.run(model.global_step)+1)\n print('saved new model to %s ' % saving_path)\n\n\nif __name__ == '__main__':\n tf.app.run()\n" }, { "path": "zhihu-text-classification-master/models/wd_5_bigru_cnn/__init__.py", "content": "# -*- coding:utf-8 -*- \n\n" }, { "path": "zhihu-text-classification-master/models/wd_5_bigru_cnn/network.py", "content": "# -*- coding:utf-8 -*-\n\nimport tensorflow as tf\nfrom tensorflow.contrib import rnn\nimport tensorflow.contrib.layers as layers\n\n\"\"\"wd_5_bigru_cnn\n两部分使用不同的 embedding， 因为RNN与CNN结构完全不同，共用embedding会降低性能。\ntitle 部分使用 bigru+attention；content 部分使用 textcnn； 两部分输出直接 concat。\n\"\"\"\n\n\nclass Settings(object):\n def __init__(self):\n self.model_name = 'wd_5_bigru_cnn'\n self.title_len = 30\n self.content_len = 150\n self.hidden_size = 256\n self.n_layer = 1\n self.filter_sizes = [2, 3, 4, 5, 7]\n self.n_filter = 256\n self.fc_hidden_size = 1024\n self.n_class = 1999\n self.summary_path = '../../summary/' + self.model_name + '/'\n self.ckpt_path = '../../ckpt/' + self.model_name + '/'\n\n\nclass BiGRU_CNN(object):\n \"\"\"\n title: inputs->bigru+attention->output_title\n content: inputs->textcnn->output_content\n concat[output_title, output_content] -> fc+bn+relu -> sigmoid_entropy.\n \"\"\"\n\n def __init__(self, W_embedding, settings):\n self.model_name = settings.model_name\n self.title_len = settings.title_len\n self.content_len = settings.content_len\n self.hidden_size = settings.hidden_size\n self.n_layer = settings.n_layer\n self.filter_sizes = settings.filter_sizes\n self.n_filter = settings.n_filter\n self.n_filter_total = self.n_filter * len(self.filter_sizes)\n self.n_class = settings.n_class\n self.fc_hidden_size = settings.fc_hidden_size\n self._global_step = tf.Variable(0, trainable=False, name='Global_Step')\n self.update_emas = list()\n # placeholders\n self._tst = tf.placeholder(tf.bool)\n self._keep_prob = tf.placeholder(tf.float32, [])\n self._batch_size = tf.placeholder(tf.int32, [])\n\n with tf.name_scope('Inputs'):\n self._X1_inputs = tf.placeholder(tf.int64, [None, self.title_len], name='X1_inputs')\n self._X2_inputs = tf.placeholder(tf.int64, [None, self.content_len], name='X2_inputs')\n self._y_inputs = tf.placeholder(tf.float32, [None, self.n_class], name='y_input')\n\n with tf.variable_scope('embedding'):\n self.title_embedding = tf.get_variable(name='title_embedding', shape=W_embedding.shape,\n initializer=tf.constant_initializer(W_embedding), trainable=True)\n self.content_embedding = tf.get_variable(name='content_embedding', shape=W_embedding.shape,\n initializer=tf.constant_initializer(W_embedding), trainable=True)\n self.embedding_size = W_embedding.shape[1]\n\n with tf.variable_scope('bigru_text'):\n output_title = self.bigru_inference(self._X1_inputs)\n\n with tf.variable_scope('cnn_content'):\n output_content = self.cnn_inference(self._X2_inputs, self.content_len)\n\n with tf.variable_scope('fc-bn-layer'):\n output = tf.concat([output_title, output_content], axis=1)\n W_fc = self.weight_variable([self.hidden_size*2 + self.n_filter_total, self.fc_hidden_size], name='Weight_fc')\n tf.summary.histogram('W_fc', W_fc)\n h_fc = tf.matmul(output, W_fc, name='h_fc')\n beta_fc = tf.Variable(tf.constant(0.1, tf.float32, shape=[self.fc_hidden_size], name=\"beta_fc\"))\n tf.summary.histogram('beta_fc', beta_fc)\n fc_bn, update_ema_fc = self.batchnorm(h_fc, beta_fc, convolutional=False)\n self.update_emas.append(update_ema_fc)\n self.fc_bn_relu = tf.nn.relu(fc_bn, name=\"relu\")\n fc_bn_drop = tf.nn.dropout(self.fc_bn_relu, self.keep_prob)\n\n with tf.variable_scope('out_layer'):\n W_out = self.weight_variable([self.fc_hidden_size, self.n_class], name='Weight_out')\n tf.summary.histogram('Weight_out', W_out)\n b_out = self.bias_variable([self.n_class], name='bias_out')\n tf.summary.histogram('bias_out', b_out)\n self._y_pred = tf.nn.xw_plus_b(fc_bn_drop, W_out, b_out, name='y_pred') # 每个类别的分数 scores\n\n with tf.name_scope('loss'):\n self._loss = tf.reduce_mean(\n tf.nn.sigmoid_cross_entropy_with_logits(logits=self._y_pred, labels=self._y_inputs))\n tf.summary.scalar('loss', self._loss)\n\n self.saver = tf.train.Saver(max_to_keep=1)\n\n @property\n def tst(self):\n return self._tst\n\n @property\n def keep_prob(self):\n return self._keep_prob\n\n @property\n def batch_size(self):\n return self._batch_size\n\n @property\n def global_step(self):\n return self._global_step\n\n @property\n def X1_inputs(self):\n return self._X1_inputs\n\n @property\n def X2_inputs(self):\n return self._X2_inputs\n\n @property\n def y_inputs(self):\n return self._y_inputs\n\n @property\n def y_pred(self):\n return self._y_pred\n\n @property\n def loss(self):\n return self._loss\n\n def weight_variable(self, shape, name):\n \"\"\"Create a weight variable with appropriate initialization.\"\"\"\n initial = tf.truncated_normal(shape, stddev=0.1)\n return tf.Variable(initial, name=name)\n\n def bias_variable(self, shape, name):\n \"\"\"Create a bias variable with appropriate initialization.\"\"\"\n initial = tf.constant(0.1, shape=shape)\n return tf.Variable(initial, name=name)\n\n def batchnorm(self, Ylogits, offset, convolutional=False):\n \"\"\"batchnormalization.\n Args:\n Ylogits: 1D向量或者是3D的卷积结果。\n num_updates: 迭代的global_step\n offset：表示beta，全局均值；在 RELU 激活中一般初始化为 0.1。\n scale：表示lambda，全局方差；在 sigmoid 激活中需要，这 RELU 激活中作用不大。\n m: 表示batch均值；v:表示batch方差。\n bnepsilon：一个很小的浮点数，防止除以 0.\n Returns:\n Ybn: 和 Ylogits 的维度一样，就是经过 Batch Normalization 处理的结果。\n update_moving_everages：更新mean和variance，主要是给最后的 test 使用。\n \"\"\"\n exp_moving_avg = tf.train.ExponentialMovingAverage(0.999, self._global_step) # adding the iteration prevents from averaging across non-existing iterations\n bnepsilon = 1e-5\n if convolutional:\n mean, variance = tf.nn.moments(Ylogits, [0, 1, 2])\n else:\n mean, variance = tf.nn.moments(Ylogits, [0])\n update_moving_everages = exp_moving_avg.apply([mean, variance])\n m = tf.cond(self.tst, lambda: exp_moving_avg.average(mean), lambda: mean)\n v = tf.cond(self.tst, lambda: exp_moving_avg.average(variance), lambda: variance)\n Ybn = tf.nn.batch_normalization(Ylogits, m, v, offset, None, bnepsilon)\n return Ybn, update_moving_everages\n\n def gru_cell(self):\n with tf.name_scope('gru_cell'):\n cell = rnn.GRUCell(self.hidden_size, reuse=tf.get_variable_scope().reuse)\n return rnn.DropoutWrapper(cell, output_keep_prob=self.keep_prob)\n\n def bi_gru(self, inputs):\n \"\"\"build the bi-GRU network. 返回个所有层的隐含状态。\"\"\"\n cells_fw = [self.gru_cell() for _ in range(self.n_layer)]\n cells_bw = [self.gru_cell() for _ in range(self.n_layer)]\n initial_states_fw = [cell_fw.zero_state(self.batch_size, tf.float32) for cell_fw in cells_fw]\n initial_states_bw = [cell_bw.zero_state(self.batch_size, tf.float32) for cell_bw in cells_bw]\n outputs, _, _ = rnn.stack_bidirectional_dynamic_rnn(cells_fw, cells_bw, inputs,\n initial_states_fw=initial_states_fw,\n initial_states_bw=initial_states_bw, dtype=tf.float32)\n return outputs\n\n def task_specific_attention(self, inputs, output_size,\n initializer=layers.xavier_initializer(),\n activation_fn=tf.tanh, scope=None):\n \"\"\"\n Performs task-specific attention reduction, using learned\n attention context vector (constant within task of interest).\n Args:\n inputs: Tensor of shape [batch_size, units, input_size]\n `input_size` must be static (known)\n `units` axis will be attended over (reduced from output)\n `batch_size` will be preserved\n output_size: Size of output's inner (feature) dimension\n Returns:\n outputs: Tensor of shape [batch_size, output_dim].\n \"\"\"\n assert len(inputs.get_shape()) == 3 and inputs.get_shape()[-1].value is not None\n with tf.variable_scope(scope or 'attention') as scope:\n # u_w, attention 向量\n attention_context_vector = tf.get_variable(name='attention_context_vector', shape=[output_size],\n initializer=initializer, dtype=tf.float32)\n # 全连接层，把 h_i 转为 u_i ， shape= [batch_size, units, input_size] -> [batch_size, units, output_size]\n input_projection = layers.fully_connected(inputs, output_size, activation_fn=activation_fn, scope=scope)\n # 输出 [batch_size, units]\n vector_attn = tf.reduce_sum(tf.multiply(input_projection, attention_context_vector), axis=2, keep_dims=True)\n attention_weights = tf.nn.softmax(vector_attn, dim=1)\n tf.summary.histogram('attention_weigths', attention_weights)\n weighted_projection = tf.multiply(inputs, attention_weights)\n outputs = tf.reduce_sum(weighted_projection, axis=1)\n return outputs # 输出 [batch_size, hidden_size*2]\n\n def bigru_inference(self, X_inputs):\n inputs = tf.nn.embedding_lookup(self.title_embedding, X_inputs)\n output_bigru = self.bi_gru(inputs)\n output_att = self.task_specific_attention(output_bigru, self.hidden_size*2)\n return output_att\n\n def cnn_inference(self, X_inputs, n_step):\n \"\"\"TextCNN 模型。\n Args:\n X_inputs: tensor.shape=(batch_size, n_step)\n Returns:\n title_outputs: tensor.shape=(batch_size, self.n_filter_total)\n \"\"\"\n inputs = tf.nn.embedding_lookup(self.content_embedding, X_inputs)\n inputs = tf.expand_dims(inputs, -1)\n pooled_outputs = list()\n for i, filter_size in enumerate(self.filter_sizes):\n with tf.variable_scope(\"conv-maxpool-%s\" % filter_size):\n # Convolution Layer\n filter_shape = [filter_size, self.embedding_size, 1, self.n_filter]\n W_filter = self.weight_variable(shape=filter_shape, name='W_filter')\n beta = self.bias_variable(shape=[self.n_filter], name='beta_filter')\n tf.summary.histogram('beta', beta)\n conv = tf.nn.conv2d(inputs, W_filter, strides=[1, 1, 1, 1], padding=\"VALID\", name=\"conv\")\n conv_bn, update_ema = self.batchnorm(conv, beta, convolutional=True)\n # Apply nonlinearity, batch norm scaling is not useful with relus\n h = tf.nn.relu(conv_bn, name=\"relu\")\n # Maxpooling over the outputs\n pooled = tf.nn.max_pool(h, ksize=[1, n_step - filter_size + 1, 1, 1],\n strides=[1, 1, 1, 1], padding='VALID', name=\"pool\")\n pooled_outputs.append(pooled)\n self.update_emas.append(update_ema)\n h_pool = tf.concat(pooled_outputs, 3)\n h_pool_flat = tf.reshape(h_pool, [-1, self.n_filter_total])\n return h_pool_flat # shape = [batch_size, self.n_filter_total]\n\n\n# test the model\ndef test():\n import numpy as np\n print('Begin testing...')\n settings = Settings()\n W_embedding = np.random.randn(50, 10)\n config = tf.ConfigProto()\n config.gpu_options.allow_growth = True\n batch_size = 128\n with tf.Session(config=config) as sess:\n model = BiGRU_CNN(W_embedding, settings)\n optimizer = tf.train.AdamOptimizer(0.001)\n train_op = optimizer.minimize(model.loss)\n update_op = tf.group(*model.update_emas)\n sess.run(tf.global_variables_initializer())\n fetch = [model.loss, model.y_pred, train_op, update_op]\n loss_list = list()\n for i in xrange(100):\n X1_batch = np.zeros((batch_size, 30), dtype=float)\n X2_batch = np.zeros((batch_size, 150), dtype=float)\n y_batch = np.zeros((batch_size, 1999), dtype=int)\n _batch_size = len(y_batch)\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: False, model.keep_prob: 0.5}\n loss, y_pred, _, _ = sess.run(fetch, feed_dict=feed_dict)\n loss_list.append(loss)\n print(i, loss)\n\nif __name__ == '__main__':\n test()\n" }, { "path": "zhihu-text-classification-master/models/wd_5_bigru_cnn/predict.py", "content": "# -*- coding:utf-8 -*-\n\nfrom __future__ import print_function\nfrom __future__ import division\nimport tensorflow as tf\nimport numpy as np\nfrom tqdm import tqdm\nimport os\nimport sys\nimport time\nimport network\n\nsys.path.append('../..')\nfrom evaluator import score_eval\n\nsettings = network.Settings()\ntitle_len = settings.title_len\nmodel_name = settings.model_name\nckpt_path = settings.ckpt_path\n\nlocal_scores_path = '../../local_scores/'\nscores_path = '../../scores/'\nif not os.path.exists(local_scores_path):\n os.makedirs(local_scores_path)\nif not os.path.exists(scores_path):\n os.makedirs(scores_path)\n\nembedding_path = '../../data/word_embedding.npy'\ndata_valid_path = '../../data/wd-data/data_valid/'\ndata_test_path = '../../data/wd-data/data_test/'\nva_batches = os.listdir(data_valid_path)\nte_batches = os.listdir(data_test_path) # batch 文件名列表\nn_va_batches = len(va_batches)\nn_te_batches = len(te_batches)\n\n\ndef get_batch(batch_id):\n \"\"\"get a batch from valid data\"\"\"\n new_batch = np.load(data_valid_path + str(batch_id) + '.npz')\n X_batch = new_batch['X']\n y_batch = new_batch['y']\n X1_batch = X_batch[:, :title_len]\n X2_batch = X_batch[:, title_len:]\n return [X1_batch, X2_batch, y_batch]\n\n\ndef get_test_batch(batch_id):\n \"\"\"get a batch from test data\"\"\"\n X_batch = np.load(data_test_path + str(batch_id) + '.npy')\n X1_batch = X_batch[:, :title_len]\n X2_batch = X_batch[:, title_len:]\n return [X1_batch, X2_batch]\n\n\ndef local_predict(sess, model):\n \"\"\"Test on the valid data.\"\"\"\n time0 = time.time()\n predict_labels_list = list() # 所有的预测结果\n marked_labels_list = list()\n predict_scores = list()\n for i in tqdm(xrange(n_va_batches)):\n [X1_batch, X2_batch, y_batch] = get_batch(i)\n marked_labels_list.extend(y_batch)\n _batch_size = len(X1_batch)\n fetches = [model.y_pred]\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n predict_labels = sess.run(fetches, feed_dict)[0]\n predict_scores.append(predict_labels)\n predict_labels = list(map(lambda label: label.argsort()[-1:-6:-1], predict_labels)) # 取最大的5个下标\n predict_labels_list.extend(predict_labels)\n predict_label_and_marked_label_list = zip(predict_labels_list, marked_labels_list)\n precision, recall, f1 = score_eval(predict_label_and_marked_label_list)\n print('Local valid p=%g, r=%g, f1=%g' % (precision, recall, f1))\n predict_scores = np.vstack(np.asarray(predict_scores))\n local_scores_name = local_scores_path + model_name + '.npy'\n np.save(local_scores_name, predict_scores)\n print('local_scores.shape=', predict_scores.shape)\n print('Writed the scores into %s, time %g s' % (local_scores_name, time.time() - time0))\n\n\ndef predict(sess, model):\n \"\"\"Test on the test data.\"\"\"\n time0 = time.time()\n predict_scores = list()\n for i in tqdm(xrange(n_te_batches)):\n [X1_batch, X2_batch] = get_test_batch(i)\n _batch_size = len(X1_batch)\n fetches = [model.y_pred]\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n predict_labels = sess.run(fetches, feed_dict)[0]\n predict_scores.append(predict_labels)\n predict_scores = np.vstack(np.asarray(predict_scores))\n scores_name = scores_path + model_name + '.npy'\n np.save(scores_name, predict_scores)\n print('scores.shape=', predict_scores.shape)\n print('Writed the scores into %s, time %g s' % (scores_name, time.time() - time0))\n\n\ndef main(_):\n if not os.path.exists(ckpt_path + 'checkpoint'):\n print('there is not saved model, please check the ckpt path')\n exit()\n print('Loading model...')\n W_embedding = np.load(embedding_path)\n config = tf.ConfigProto()\n config.gpu_options.allow_growth = True\n with tf.Session(config=config) as sess:\n model = network.BiGRU_CNN(W_embedding, settings)\n model.saver.restore(sess, tf.train.latest_checkpoint(ckpt_path))\n print('Local predicting...')\n local_predict(sess, model)\n print('Test predicting...')\n predict(sess, model)\n\n\nif __name__ == '__main__':\n tf.app.run()\n" }, { "path": "zhihu-text-classification-master/models/wd_5_bigru_cnn/train.py", "content": "# -*- coding:utf-8 -*-\n\nfrom __future__ import print_function\nfrom __future__ import division\nimport tensorflow as tf\nimport numpy as np\nfrom tqdm import tqdm\nimport os\nimport sys\nimport shutil\nimport time\nimport network\n\nsys.path.append('../..')\nfrom data_helpers import to_categorical\nfrom evaluator import score_eval\n\nflags = tf.flags\nflags.DEFINE_bool('is_retrain', False, 'if is_retrain is true, not rebuild the summary')\nflags.DEFINE_integer('max_epoch', 1, 'update the embedding after max_epoch, default: 1')\nflags.DEFINE_integer('max_max_epoch', 6, 'all training epoches, default: 6')\nflags.DEFINE_float('lr', 8e-4, 'initial learning rate, default: 8e-4')\nflags.DEFINE_float('decay_rate', 0.75, 'decay rate, default: 0.75')\nflags.DEFINE_float('keep_prob', 0.5, 'keep_prob for training, default: 0.5')\n# 正式\nflags.DEFINE_integer('decay_step', 15000, 'decay_step, default: 15000')\nflags.DEFINE_integer('valid_step', 10000, 'valid_step, default: 10000')\nflags.DEFINE_float('last_f1', 0.38, 'if valid_f1 > last_f1, save new model. default: 0.40')\n\n# 测试\n# flags.DEFINE_integer('decay_step', 1000, 'decay_step, default: 1000')\n# flags.DEFINE_integer('valid_step', 500, 'valid_step, default: 500')\n# flags.DEFINE_float('last_f1', 0.10, 'if valid_f1 > last_f1, save new model. default: 0.10')\nFLAGS = flags.FLAGS\n\nlr = FLAGS.lr\nlast_f1 = FLAGS.last_f1\nsettings = network.Settings()\ntitle_len = settings.title_len\nsummary_path = settings.summary_path\nckpt_path = settings.ckpt_path\nmodel_path = ckpt_path + 'model.ckpt'\n\nembedding_path = '../../data/word_embedding.npy'\ndata_train_path = '../../data/wd-data/data_train/'\ndata_valid_path = '../../data/wd-data/data_valid/'\ntr_batches = os.listdir(data_train_path) # batch 文件名列表\nva_batches = os.listdir(data_valid_path)\nn_tr_batches = len(tr_batches)\nn_va_batches = len(va_batches)\n\n# 测试\n# n_tr_batches = 1000\n# n_va_batches = 50\n\n\ndef get_batch(data_path, batch_id):\n \"\"\"get a batch from data_path\"\"\"\n new_batch = np.load(data_path + str(batch_id) + '.npz')\n X_batch = new_batch['X']\n y_batch = new_batch['y']\n X1_batch = X_batch[:, :title_len]\n X2_batch = X_batch[:, title_len:]\n return [X1_batch, X2_batch, y_batch]\n\n\ndef valid_epoch(data_path, sess, model):\n \"\"\"Test on the valid data.\"\"\"\n va_batches = os.listdir(data_path)\n n_va_batches = len(va_batches)\n _costs = 0.0\n predict_labels_list = list() # 所有的预测结果\n marked_labels_list = list()\n for i in range(n_va_batches):\n [X1_batch, X2_batch, y_batch] = get_batch(data_path, i)\n marked_labels_list.extend(y_batch)\n y_batch = to_categorical(y_batch)\n _batch_size = len(y_batch)\n fetches = [model.loss, model.y_pred]\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n _cost, predict_labels = sess.run(fetches, feed_dict)\n _costs += _cost\n predict_labels = (map(lambda label: label.argsort()[-1:-6:-1], predict_labels)) # 取最大的5个下标\n predict_labels_list.extend(predict_labels)\n predict_label_and_marked_label_list = zip(predict_labels_list, marked_labels_list)\n precision, recall, f1 = score_eval(predict_label_and_marked_label_list)\n mean_cost = _costs / n_va_batches\n return mean_cost, precision, recall, f1\n\n\ndef train_epoch(data_path, sess, model, train_fetches, valid_fetches, train_writer, test_writer):\n global last_f1\n global lr\n time0 = time.time()\n batch_indexs = np.random.permutation(n_tr_batches) # shuffle the training data\n for batch in tqdm(range(n_tr_batches)):\n global_step = sess.run(model.global_step)\n if 0 == (global_step + 1) % FLAGS.valid_step:\n valid_cost, precision, recall, f1 = valid_epoch(data_valid_path, sess, model)\n print('Global_step=%d: valid cost=%g; p=%g, r=%g, f1=%g, time=%g s' % (\n global_step, valid_cost, precision, recall, f1, time.time() - time0))\n time0 = time.time()\n if f1 > last_f1:\n last_f1 = f1\n saving_path = model.saver.save(sess, model_path, global_step+1)\n print('saved new model to %s ' % saving_path)\n # training\n batch_id = batch_indexs[batch]\n [X1_batch, X2_batch, y_batch] = get_batch(data_train_path, batch_id)\n y_batch = to_categorical(y_batch)\n _batch_size = len(y_batch)\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: False, model.keep_prob: FLAGS.keep_prob}\n summary, _cost, _, _ = sess.run(train_fetches, feed_dict) # the cost is the mean cost of one batch\n # valid per 500 steps\n if 0 == (global_step + 1) % 500:\n train_writer.add_summary(summary, global_step)\n batch_id = np.random.randint(0, n_va_batches) # 随机选一个验证batch\n [X1_batch, X2_batch, y_batch] = get_batch(data_valid_path, batch_id)\n y_batch = to_categorical(y_batch)\n _batch_size = len(y_batch)\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n summary, _cost = sess.run(valid_fetches, feed_dict)\n test_writer.add_summary(summary, global_step)\n\n\ndef main(_):\n global ckpt_path\n global last_f1\n if not os.path.exists(ckpt_path):\n os.makedirs(ckpt_path)\n if not os.path.exists(summary_path):\n os.makedirs(summary_path)\n elif not FLAGS.is_retrain: # 重新训练本模型，删除以前的 summary\n shutil.rmtree(summary_path)\n os.makedirs(summary_path)\n if not os.path.exists(summary_path):\n os.makedirs(summary_path)\n\n print('1.Loading data...')\n W_embedding = np.load(embedding_path)\n print('training sample_num = %d' % n_tr_batches)\n print('valid sample_num = %d' % n_va_batches)\n\n # Initial or restore the model\n print('2.Building model...')\n config = tf.ConfigProto()\n config.gpu_options.allow_growth = True\n with tf.Session(config=config) as sess:\n model = network.BiGRU_CNN(W_embedding, settings)\n with tf.variable_scope('training_ops') as vs:\n learning_rate = tf.train.exponential_decay(FLAGS.lr, model.global_step, FLAGS.decay_step,\n FLAGS.decay_rate, staircase=True)\n # two optimizer: op1, update embedding; op2, do not update embedding.\n with tf.variable_scope('Optimizer1'):\n tvars1 = tf.trainable_variables()\n grads1 = tf.gradients(model.loss, tvars1)\n optimizer1 = tf.train.AdamOptimizer(learning_rate=learning_rate)\n train_op1 = optimizer1.apply_gradients(zip(grads1, tvars1),\n global_step=model.global_step)\n with tf.variable_scope('Optimizer2'):\n tvars2 = [tvar for tvar in tvars1 if 'embedding' not in tvar.name]\n grads2 = tf.gradients(model.loss, tvars2)\n optimizer2 = tf.train.AdamOptimizer(learning_rate=learning_rate)\n train_op2 = optimizer2.apply_gradients(zip(grads2, tvars2),\n global_step=model.global_step)\n update_op = tf.group(*model.update_emas)\n merged = tf.summary.merge_all() # summary\n train_writer = tf.summary.FileWriter(summary_path + 'train', sess.graph)\n test_writer = tf.summary.FileWriter(summary_path + 'test')\n training_ops = [v for v in tf.global_variables() if v.name.startswith(vs.name+'/')]\n\n # 如果已经保存过模型，导入上次的模型\n if os.path.exists(ckpt_path + \"checkpoint\"):\n print(\"Restoring Variables from Checkpoint...\")\n model.saver.restore(sess, tf.train.latest_checkpoint(ckpt_path))\n last_valid_cost, precision, recall, last_f1 = valid_epoch(data_valid_path, sess, model)\n print(' valid cost=%g; p=%g, r=%g, f1=%g' % (last_valid_cost, precision, recall, last_f1))\n sess.run(tf.variables_initializer(training_ops))\n train_op2 = train_op1\n else:\n print('Initializing Variables...')\n sess.run(tf.global_variables_initializer())\n\n print('3.Begin training...')\n print('max_epoch=%d, max_max_epoch=%d' % (FLAGS.max_epoch, FLAGS.max_max_epoch))\n train_op = train_op1\n for epoch in range(FLAGS.max_max_epoch):\n global_step = sess.run(model.global_step)\n print('Global step %d, lr=%g' % (global_step, sess.run(learning_rate)))\n if epoch == FLAGS.max_epoch: # update the embedding\n train_op = train_op1\n train_fetches = [merged, model.loss, train_op, update_op]\n valid_fetches = [merged, model.loss]\n train_epoch(data_train_path, sess, model, train_fetches, valid_fetches, train_writer, test_writer)\n # 最后再做一次验证\n valid_cost, precision, recall, f1 = valid_epoch(data_valid_path, sess, model)\n print('END.Global_step=%d: valid cost=%g; p=%g, r=%g, f1=%g' % (\n sess.run(model.global_step), valid_cost, precision, recall, f1))\n if f1 > last_f1: # save the better model\n saving_path = model.saver.save(sess, model_path, sess.run(model.global_step)+1)\n print('saved new model to %s ' % saving_path)\n\n\nif __name__ == '__main__':\n tf.app.run()\n" }, { "path": "zhihu-text-classification-master/models/wd_6_rcnn/__init__.py", "content": "# -*- coding:utf-8 -*- \n\n" }, { "path": "zhihu-text-classification-master/models/wd_6_rcnn/network.py", "content": "# -*- coding:utf-8 -*-\n\nimport tensorflow as tf\nfrom tensorflow.contrib import rnn\nimport tensorflow.contrib.layers as layers\n\n\"\"\"wd_6_rcnn\n在论文 Recurrent Convolutional Neural Networks for Text Classification 中。\n使用 BiRNN 处理，将每个时刻的隐藏状态和原输入拼起来，在进行 max_pooling 操作。\n这里有些不同，首先也是使用 bigru 得到每个时刻的，将每个时刻的隐藏状态和原输入拼起来；\n然后使用输入到 TextCNN 网络中。\n\"\"\"\n\n\nclass Settings(object):\n def __init__(self):\n self.model_name = \"wd_6_rcnn\"\n self.title_len = 30\n self.content_len = 150\n self.hidden_size = 256\n self.n_layer = 1\n self.filter_sizes = [2, 3, 4, 5, 7]\n self.n_filter = 256\n self.fc_hidden_size = 1024\n self.n_class = 1999\n self.summary_path = '../../summary/' + self.model_name + '/'\n self.ckpt_path = '../../ckpt/' + self.model_name + '/'\n\n\nclass RCNN(object):\n def __init__(self, W_embedding, settings):\n self.model_name = settings.model_name\n self.title_len = settings.title_len\n self.content_len = settings.content_len\n self.hidden_size = settings.hidden_size\n self.n_layer = settings.n_layer\n self.filter_sizes = settings.filter_sizes\n self.n_filter = settings.n_filter\n self.n_filter_total = self.n_filter * len(self.filter_sizes)\n self.n_class = settings.n_class\n self.fc_hidden_size = settings.fc_hidden_size\n self._global_step = tf.Variable(0, trainable=False, name='Global_Step')\n self.update_emas = list()\n # placeholders\n self._tst = tf.placeholder(tf.bool)\n self._keep_prob = tf.placeholder(tf.float32, [])\n self._batch_size = tf.placeholder(tf.int32, [])\n\n with tf.name_scope('Inputs'):\n self._X1_inputs = tf.placeholder(tf.int64, [None, self.title_len], name='X1_inputs')\n self._X2_inputs = tf.placeholder(tf.int64, [None, self.content_len], name='X2_inputs')\n self._y_inputs = tf.placeholder(tf.float32, [None, self.n_class], name='y_input')\n\n with tf.variable_scope('embedding'):\n self.embedding = tf.get_variable(name='embedding', shape=W_embedding.shape,\n initializer=tf.constant_initializer(W_embedding), trainable=True)\n self.embedding_size = W_embedding.shape[1]\n\n with tf.variable_scope('rcnn_text'):\n output_title = self.rcnn_inference(self._X1_inputs, self.title_len)\n\n with tf.variable_scope('rcnn_content'):\n output_content = self.rcnn_inference(self._X2_inputs, self.content_len)\n\n with tf.variable_scope('fc-bn-layer'):\n output = tf.concat([output_title, output_content], axis=1)\n W_fc = self.weight_variable([self.n_filter_total*2, self.fc_hidden_size],\n name='Weight_fc')\n tf.summary.histogram('W_fc', W_fc)\n h_fc = tf.matmul(output, W_fc, name='h_fc')\n beta_fc = tf.Variable(tf.constant(0.1, tf.float32, shape=[self.fc_hidden_size], name=\"beta_fc\"))\n tf.summary.histogram('beta_fc', beta_fc)\n fc_bn, update_ema_fc = self.batchnorm(h_fc, beta_fc, convolutional=False)\n self.update_emas.append(update_ema_fc)\n self.fc_bn_relu = tf.nn.relu(fc_bn, name=\"relu\")\n fc_bn_drop = tf.nn.dropout(self.fc_bn_relu, self.keep_prob)\n\n with tf.variable_scope('out_layer'):\n W_out = self.weight_variable([self.fc_hidden_size, self.n_class], name='Weight_out')\n tf.summary.histogram('Weight_out', W_out)\n b_out = self.bias_variable([self.n_class], name='bias_out')\n tf.summary.histogram('bias_out', b_out)\n self._y_pred = tf.nn.xw_plus_b(fc_bn_drop, W_out, b_out, name='y_pred') # 每个类别的分数 scores\n\n with tf.name_scope('loss'):\n self._loss = tf.reduce_mean(\n tf.nn.sigmoid_cross_entropy_with_logits(logits=self._y_pred, labels=self._y_inputs))\n tf.summary.scalar('loss', self._loss)\n\n self.saver = tf.train.Saver(max_to_keep=1)\n\n @property\n def tst(self):\n return self._tst\n\n @property\n def keep_prob(self):\n return self._keep_prob\n\n @property\n def batch_size(self):\n return self._batch_size\n\n @property\n def global_step(self):\n return self._global_step\n\n @property\n def X1_inputs(self):\n return self._X1_inputs\n\n @property\n def X2_inputs(self):\n return self._X2_inputs\n\n @property\n def y_inputs(self):\n return self._y_inputs\n\n @property\n def y_pred(self):\n return self._y_pred\n\n @property\n def loss(self):\n return self._loss\n\n def weight_variable(self, shape, name):\n \"\"\"Create a weight variable with appropriate initialization.\"\"\"\n initial = tf.truncated_normal(shape, stddev=0.1)\n return tf.Variable(initial, name=name)\n\n def bias_variable(self, shape, name):\n \"\"\"Create a bias variable with appropriate initialization.\"\"\"\n initial = tf.constant(0.1, shape=shape)\n return tf.Variable(initial, name=name)\n\n def batchnorm(self, Ylogits, offset, convolutional=False):\n \"\"\"batchnormalization.\n Args:\n Ylogits: 1D向量或者是3D的卷积结果。\n num_updates: 迭代的global_step\n offset：表示beta，全局均值；在 RELU 激活中一般初始化为 0.1。\n scale：表示lambda，全局方差；在 sigmoid 激活中需要，这 RELU 激活中作用不大。\n m: 表示batch均值；v:表示batch方差。\n bnepsilon：一个很小的浮点数，防止除以 0.\n Returns:\n Ybn: 和 Ylogits 的维度一样，就是经过 Batch Normalization 处理的结果。\n update_moving_everages：更新mean和variance，主要是给最后的 test 使用。\n \"\"\"\n exp_moving_avg = tf.train.ExponentialMovingAverage(0.999,\n self._global_step) # adding the iteration prevents from averaging across non-existing iterations\n bnepsilon = 1e-5\n if convolutional:\n mean, variance = tf.nn.moments(Ylogits, [0, 1, 2])\n else:\n mean, variance = tf.nn.moments(Ylogits, [0])\n update_moving_everages = exp_moving_avg.apply([mean, variance])\n m = tf.cond(self.tst, lambda: exp_moving_avg.average(mean), lambda: mean)\n v = tf.cond(self.tst, lambda: exp_moving_avg.average(variance), lambda: variance)\n Ybn = tf.nn.batch_normalization(Ylogits, m, v, offset, None, bnepsilon)\n return Ybn, update_moving_everages\n\n def gru_cell(self):\n with tf.name_scope('gru_cell'):\n cell = rnn.GRUCell(self.hidden_size, reuse=tf.get_variable_scope().reuse)\n return rnn.DropoutWrapper(cell, output_keep_prob=self.keep_prob)\n\n def bi_gru(self, X_inputs):\n \"\"\"build the bi-GRU network. Return the encoder represented vector.\n X_inputs: [batch_size, n_step]\n n_step: 句子的词数量；或者文档的句子数。\n outputs: [fw_state, embeddings, bw_state], shape=[batch_size, hidden_size+embedding_size+hidden_size]\n \"\"\"\n inputs = tf.nn.embedding_lookup(self.embedding, X_inputs) # [batch_size, n_step, embedding_size]\n cells_fw = [self.gru_cell() for _ in range(self.n_layer)]\n cells_bw = [self.gru_cell() for _ in range(self.n_layer)]\n initial_states_fw = [cell_fw.zero_state(self.batch_size, tf.float32) for cell_fw in cells_fw]\n initial_states_bw = [cell_bw.zero_state(self.batch_size, tf.float32) for cell_bw in cells_bw]\n outputs, _, _ = rnn.stack_bidirectional_dynamic_rnn(cells_fw, cells_bw, inputs,\n initial_states_fw = initial_states_fw, initial_states_bw = initial_states_bw, dtype=tf.float32)\n hidden_outputs = tf.concat([outputs, inputs], axis=2)\n return hidden_outputs # shape =[seg_num, n_steps, hidden_size*2+embedding_size]\n\n def textcnn(self, cnn_inputs, n_step):\n \"\"\"build the TextCNN network. Return the h_drop\"\"\"\n # cnn_inputs.shape = [batchsize, n_step, hidden_size*2+embedding_size]\n inputs = tf.expand_dims(cnn_inputs, -1)\n pooled_outputs = list()\n for i, filter_size in enumerate(self.filter_sizes):\n with tf.variable_scope(\"conv-maxpool-%s\" % filter_size):\n # Convolution Layer\n filter_shape = [filter_size, self.hidden_size*2+self.embedding_size, 1, self.n_filter]\n W_filter = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name=\"W_filter\")\n beta = tf.Variable(tf.constant(0.1, tf.float32, shape=[self.n_filter], name=\"beta\"))\n tf.summary.histogram('beta', beta)\n conv = tf.nn.conv2d(inputs, W_filter, strides=[1, 1, 1, 1], padding=\"VALID\", name=\"conv\")\n conv_bn, update_ema = self.batchnorm(conv, beta, convolutional=True) # 在激活层前面加 BN\n # Apply nonlinearity, batch norm scaling is not useful with relus\n h = tf.nn.relu(conv_bn, name=\"relu\")\n # Maxpooling over the outputs\n pooled = tf.nn.max_pool(h,ksize=[1, n_step - filter_size + 1, 1, 1],\n strides=[1, 1, 1, 1],padding='VALID',name=\"pool\")\n pooled_outputs.append(pooled)\n self.update_emas.append(update_ema)\n h_pool = tf.concat(pooled_outputs, 3)\n h_pool_flat = tf.reshape(h_pool, [-1, self.n_filter_total])\n return h_pool_flat # shape = [batch_size, n_filter_total]\n\n def rcnn_inference(self, X_inputs, n_step):\n output_bigru = self.bi_gru(X_inputs)\n output_cnn = self.textcnn(output_bigru, n_step)\n return output_cnn # shape = [batch_size, n_filter_total]\n\n\n# test the model\ndef test():\n import numpy as np\n print('Begin testing...')\n settings = Settings()\n W_embedding = np.random.randn(50, 10)\n config = tf.ConfigProto()\n config.gpu_options.allow_growth = True\n batch_size = 128\n with tf.Session(config=config) as sess:\n model = RCNN(W_embedding, settings)\n optimizer = tf.train.AdamOptimizer(0.001)\n train_op = optimizer.minimize(model.loss)\n update_op = tf.group(*model.update_emas)\n sess.run(tf.global_variables_initializer())\n fetch = [model.loss, model.y_pred, train_op, update_op]\n loss_list = list()\n for i in xrange(100):\n X1_batch = np.zeros((batch_size, 30), dtype=float)\n X2_batch = np.zeros((batch_size, 150), dtype=float)\n y_batch = np.zeros((batch_size, 1999), dtype=int)\n _batch_size = len(y_batch)\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: False, model.keep_prob: 0.5}\n loss, y_pred, _, _ = sess.run(fetch, feed_dict=feed_dict)\n loss_list.append(loss)\n print(i, loss)\n\n\nif __name__ == '__main__':\n test()\n" }, { "path": "zhihu-text-classification-master/models/wd_6_rcnn/predict.py", "content": "# -*- coding:utf-8 -*-\n\nfrom __future__ import print_function\nfrom __future__ import division\nimport tensorflow as tf\nimport numpy as np\nfrom tqdm import tqdm\nimport os\nimport sys\nimport time\nimport network\n\nsys.path.append('../..')\nfrom evaluator import score_eval\n\nsettings = network.Settings()\ntitle_len = settings.title_len\nmodel_name = settings.model_name\nckpt_path = settings.ckpt_path\n\nlocal_scores_path = '../../local_scores/'\nscores_path = '../../scores/'\nif not os.path.exists(local_scores_path):\n os.makedirs(local_scores_path)\nif not os.path.exists(scores_path):\n os.makedirs(scores_path)\n\nembedding_path = '../../data/word_embedding.npy'\ndata_valid_path = '../../data/wd-data/data_valid/'\ndata_test_path = '../../data/wd-data/data_test/'\nva_batches = os.listdir(data_valid_path)\nte_batches = os.listdir(data_test_path) # batch 文件名列表\nn_va_batches = len(va_batches)\nn_te_batches = len(te_batches)\n\n\ndef get_batch(batch_id):\n \"\"\"get a batch from valid data\"\"\"\n new_batch = np.load(data_valid_path + str(batch_id) + '.npz')\n X_batch = new_batch['X']\n y_batch = new_batch['y']\n X1_batch = X_batch[:, :title_len]\n X2_batch = X_batch[:, title_len:]\n return [X1_batch, X2_batch, y_batch]\n\n\ndef get_test_batch(batch_id):\n \"\"\"get a batch from test data\"\"\"\n X_batch = np.load(data_test_path + str(batch_id) + '.npy')\n X1_batch = X_batch[:, :title_len]\n X2_batch = X_batch[:, title_len:]\n return [X1_batch, X2_batch]\n\n\ndef local_predict(sess, model):\n \"\"\"Test on the valid data.\"\"\"\n time0 = time.time()\n predict_labels_list = list() # 所有的预测结果\n marked_labels_list = list()\n predict_scores = list()\n for i in tqdm(xrange(n_va_batches)):\n [X1_batch, X2_batch, y_batch] = get_batch(i)\n marked_labels_list.extend(y_batch)\n _batch_size = len(X1_batch)\n fetches = [model.y_pred]\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n predict_labels = sess.run(fetches, feed_dict)[0]\n predict_scores.append(predict_labels)\n predict_labels = list(map(lambda label: label.argsort()[-1:-6:-1], predict_labels)) # 取最大的5个下标\n predict_labels_list.extend(predict_labels)\n predict_label_and_marked_label_list = zip(predict_labels_list, marked_labels_list)\n precision, recall, f1 = score_eval(predict_label_and_marked_label_list)\n print('Local valid p=%g, r=%g, f1=%g' % (precision, recall, f1))\n predict_scores = np.vstack(np.asarray(predict_scores))\n local_scores_name = local_scores_path + model_name + '.npy'\n np.save(local_scores_name, predict_scores)\n print('local_scores.shape=', predict_scores.shape)\n print('Writed the scores into %s, time %g s' % (local_scores_name, time.time() - time0))\n\n\ndef predict(sess, model):\n \"\"\"Test on the test data.\"\"\"\n time0 = time.time()\n predict_scores = list()\n for i in tqdm(xrange(n_te_batches)):\n [X1_batch, X2_batch] = get_test_batch(i)\n _batch_size = len(X1_batch)\n fetches = [model.y_pred]\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n predict_labels = sess.run(fetches, feed_dict)[0]\n predict_scores.append(predict_labels)\n predict_scores = np.vstack(np.asarray(predict_scores))\n scores_name = scores_path + model_name + '.npy'\n np.save(scores_name, predict_scores)\n print('scores.shape=', predict_scores.shape)\n print('Writed the scores into %s, time %g s' % (scores_name, time.time() - time0))\n\n\ndef main(_):\n if not os.path.exists(ckpt_path + 'checkpoint'):\n print('there is not saved model, please check the ckpt path')\n exit()\n print('Loading model...')\n W_embedding = np.load(embedding_path)\n config = tf.ConfigProto()\n config.gpu_options.allow_growth = True\n with tf.Session(config=config) as sess:\n model = network.RCNN(W_embedding, settings)\n model.saver.restore(sess, tf.train.latest_checkpoint(ckpt_path))\n print('Local predicting...')\n local_predict(sess, model)\n print('Test predicting...')\n predict(sess, model)\n\n\nif __name__ == '__main__':\n tf.app.run()\n" }, { "path": "zhihu-text-classification-master/models/wd_6_rcnn/train.py", "content": "# -*- coding:utf-8 -*-\n\nfrom __future__ import print_function\nfrom __future__ import division\nimport tensorflow as tf\nimport numpy as np\nfrom tqdm import tqdm\nimport os\nimport sys\nimport shutil\nimport time\nimport network\n\nsys.path.append('../..')\nfrom data_helpers import to_categorical\nfrom evaluator import score_eval\n\nflags = tf.flags\nflags.DEFINE_bool('is_retrain', False, 'if is_retrain is true, not rebuild the summary')\nflags.DEFINE_integer('max_epoch', 1, 'update the embedding after max_epoch, default: 1')\nflags.DEFINE_integer('max_max_epoch', 6, 'all training epoches, default: 6')\nflags.DEFINE_float('lr', 8e-4, 'initial learning rate, default: 8e-4')\nflags.DEFINE_float('decay_rate', 0.75, 'decay rate, default: 0.75')\nflags.DEFINE_float('keep_prob', 0.5, 'keep_prob for training, default: 0.5')\n# 正式\nflags.DEFINE_integer('decay_step', 15000, 'decay_step, default: 15000')\nflags.DEFINE_integer('valid_step', 10000, 'valid_step, default: 10000')\nflags.DEFINE_float('last_f1', 0.38, 'if valid_f1 > last_f1, save new model. default: 0.40')\n\n# 测试\n# flags.DEFINE_integer('decay_step', 1000, 'decay_step, default: 1000')\n# flags.DEFINE_integer('valid_step', 500, 'valid_step, default: 500')\n# flags.DEFINE_float('last_f1', 0.10, 'if valid_f1 > last_f1, save new model. default: 0.10')\nFLAGS = flags.FLAGS\n\nlr = FLAGS.lr\nlast_f1 = FLAGS.last_f1\nsettings = network.Settings()\ntitle_len = settings.title_len\nsummary_path = settings.summary_path\nckpt_path = settings.ckpt_path\nmodel_path = ckpt_path + 'model.ckpt'\n\nembedding_path = '../../data/word_embedding.npy'\ndata_train_path = '../../data/wd-data/data_train/'\ndata_valid_path = '../../data/wd-data/data_valid/'\ntr_batches = os.listdir(data_train_path) # batch 文件名列表\nva_batches = os.listdir(data_valid_path)\nn_tr_batches = len(tr_batches)\nn_va_batches = len(va_batches)\n\n# 测试\n# n_tr_batches = 1000\n# n_va_batches = 50\n\n\ndef get_batch(data_path, batch_id):\n \"\"\"get a batch from data_path\"\"\"\n new_batch = np.load(data_path + str(batch_id) + '.npz')\n X_batch = new_batch['X']\n y_batch = new_batch['y']\n X1_batch = X_batch[:, :title_len]\n X2_batch = X_batch[:, title_len:]\n return [X1_batch, X2_batch, y_batch]\n\n\ndef valid_epoch(data_path, sess, model):\n \"\"\"Test on the valid data.\"\"\"\n va_batches = os.listdir(data_path)\n n_va_batches = len(va_batches)\n _costs = 0.0\n predict_labels_list = list() # 所有的预测结果\n marked_labels_list = list()\n for i in range(n_va_batches):\n [X1_batch, X2_batch, y_batch] = get_batch(data_path, i)\n marked_labels_list.extend(y_batch)\n y_batch = to_categorical(y_batch)\n _batch_size = len(y_batch)\n fetches = [model.loss, model.y_pred]\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n _cost, predict_labels = sess.run(fetches, feed_dict)\n _costs += _cost\n predict_labels = list(map(lambda label: label.argsort()[-1:-6:-1], predict_labels)) # 取最大的5个下标\n predict_labels_list.extend(predict_labels)\n predict_label_and_marked_label_list = zip(predict_labels_list, marked_labels_list)\n precision, recall, f1 = score_eval(predict_label_and_marked_label_list)\n mean_cost = _costs / n_va_batches\n return mean_cost, precision, recall, f1\n\n\ndef train_epoch(data_path, sess, model, train_fetches, valid_fetches, train_writer, test_writer):\n global last_f1\n global lr\n time0 = time.time()\n batch_indexs = np.random.permutation(n_tr_batches) # shuffle the training data\n for batch in tqdm(range(n_tr_batches)):\n global_step = sess.run(model.global_step)\n if 0 == (global_step + 1) % FLAGS.valid_step:\n valid_cost, precision, recall, f1 = valid_epoch(data_valid_path, sess, model)\n print('Global_step=%d: valid cost=%g; p=%g, r=%g, f1=%g, time=%g s' % (\n global_step, valid_cost, precision, recall, f1, time.time() - time0))\n time0 = time.time()\n if f1 > last_f1:\n last_f1 = f1\n saving_path = model.saver.save(sess, model_path, global_step+1)\n print('saved new model to %s ' % saving_path)\n # training\n batch_id = batch_indexs[batch]\n [X1_batch, X2_batch, y_batch] = get_batch(data_train_path, batch_id)\n y_batch = to_categorical(y_batch)\n _batch_size = len(y_batch)\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: False, model.keep_prob: FLAGS.keep_prob}\n summary, _cost, _, _ = sess.run(train_fetches, feed_dict) # the cost is the mean cost of one batch\n # valid per 500 steps\n if 0 == (global_step + 1) % 500:\n train_writer.add_summary(summary, global_step)\n batch_id = np.random.randint(0, n_va_batches) # 随机选一个验证batch\n [X1_batch, X2_batch, y_batch] = get_batch(data_valid_path, batch_id)\n y_batch = to_categorical(y_batch)\n _batch_size = len(y_batch)\n feed_dict = {model.X1_inputs: X1_batch, model.X2_inputs: X2_batch, model.y_inputs: y_batch,\n model.batch_size: _batch_size, model.tst: True, model.keep_prob: 1.0}\n summary, _cost = sess.run(valid_fetches, feed_dict)\n test_writer.add_summary(summary, global_step)\n\n\ndef main(_):\n global ckpt_path\n global last_f1\n if not os.path.exists(ckpt_path):\n os.makedirs(ckpt_path)\n if not os.path.exists(summary_path):\n os.makedirs(summary_path)\n elif not FLAGS.is_retrain: # 重新训练本模型，删除以前的 summary\n shutil.rmtree(summary_path)\n os.makedirs(summary_path)\n if not os.path.exists(summary_path):\n os.makedirs(summary_path)\n\n print('1.Loading data...')\n W_embedding = np.load(embedding_path)\n print('training sample_num = %d' % n_tr_batches)\n print('valid sample_num = %d' % n_va_batches)\n\n # Initial or restore the model\n print('2.Building model...')\n config = tf.ConfigProto()\n config.gpu_options.allow_growth = True\n with tf.Session(config=config) as sess:\n model = network.RCNN(W_embedding, settings)\n with tf.variable_scope('training_ops') as vs:\n learning_rate = tf.train.exponential_decay(FLAGS.lr, model.global_step, FLAGS.decay_step,\n FLAGS.decay_rate, staircase=True)\n # two optimizer: op1, update embedding; op2, do not update embedding.\n with tf.variable_scope('Optimizer1'):\n tvars1 = tf.trainable_variables()\n grads1 = tf.gradients(model.loss, tvars1)\n optimizer1 = tf.train.AdamOptimizer(learning_rate=learning_rate)\n train_op1 = optimizer1.apply_gradients(zip(grads1, tvars1),\n global_step=model.global_step)\n with tf.variable_scope('Optimizer2'):\n tvars2 = [tvar for tvar in tvars1 if 'embedding' not in tvar.name]\n grads2 = tf.gradients(model.loss, tvars2)\n optimizer2 = tf.train.AdamOptimizer(learning_rate=learning_rate)\n train_op2 = optimizer2.apply_gradients(zip(grads2, tvars2),\n global_step=model.global_step)\n update_op = tf.group(*model.update_emas)\n merged = tf.summary.merge_all() # summary\n train_writer = tf.summary.FileWriter(summary_path + 'train', sess.graph)\n test_writer = tf.summary.FileWriter(summary_path + 'test')\n training_ops = [v for v in tf.global_variables() if v.name.startswith(vs.name+'/')]\n\n # 如果已经保存过模型，导入上次的模型\n if os.path.exists(ckpt_path + \"checkpoint\"):\n print(\"Restoring Variables from Checkpoint...\")\n model.saver.restore(sess, tf.train.latest_checkpoint(ckpt_path))\n last_valid_cost, precision, recall, last_f1 = valid_epoch(data_valid_path, sess, model)\n print(' valid cost=%g; p=%g, r=%g, f1=%g' % (last_valid_cost, precision, recall, last_f1))\n sess.run(tf.variables_initializer(training_ops))\n train_op2 = train_op1\n else:\n print('Initializing Variables...')\n sess.run(tf.global_variables_initializer())\n\n print('3.Begin training...')\n print('max_epoch=%d, max_max_epoch=%d' % (FLAGS.max_epoch, FLAGS.max_max_epoch))\n train_op = train_op2\n for epoch in range(FLAGS.max_max_epoch):\n global_step = sess.run(model.global_step)\n print('Global step %d, lr=%g' % (global_step, sess.run(learning_rate)))\n if epoch == FLAGS.max_epoch: # update the embedding\n train_op = train_op1\n train_fetches = [merged, model.loss, train_op, update_op]\n valid_fetches = [merged, model.loss]\n train_epoch(data_train_path, sess, model, train_fetches, valid_fetches, train_writer, test_writer)\n # 最后再做一次验证\n valid_cost, precision, recall, f1 = valid_epoch(data_valid_path, sess, model)\n print('END.Global_step=%d: valid cost=%g; p=%g, r=%g, f1=%g' % (\n sess.run(model.global_step), valid_cost, precision, recall, f1))\n if f1 > last_f1: # save the better model\n saving_path = model.saver.save(sess, model_path, sess.run(model.global_step)+1)\n print('saved new model to %s ' % saving_path)\n\n\nif __name__ == '__main__':\n tf.app.run()\n" } ]