[ { "path": "README.md", "content": "# Document Classification\nThis code implements a simple CNN model for document classification with tensorflow.\n\n# Model Structure\n![model.png](https://github.com/MRliujiaxin/DocumentClassification/raw/master/model.png)\n\n# Requirements\n- Python: 2.7\n- Tensorflow: 1.0.0\n- Numpy: 1.12.1\n- sklearn: 0.18.1\n- gensim: 1.0.1\n- pickle\n" }, { "path": "code/Data/corpus/corpus download link", "content": "数据下载地址：http://competition.ai100.com.cn/html/game_det.html?id=24\n\n将分词/词性标注后的文件命名为training.seg.csv和testing.seg.csv，放置到当前目录下。\n\n处理后的文本如下所示:\n \"公司/n 是/vshi 经/p 批准/v 依法/d 从事/vi 融资/vi 性/ng 担保/vn 业务/n 的/ude1 ...\"\n" }, { "path": "code/README", "content": "下载数据到Data/corpus目录下，并做分词和词性标注处理，再按以下顺序执行:\n train_w2v_model.py -> prepare_data.py -> model_dc.py\n\n\n文件说明:\n\n1. 分词、词性标注采用中科院NLPIR，处理后文件:\n ./Data/corpus/training.seg.csv\n ./Data/corpus/testing.seg.csv\n\n2. configuration.py\n 配置文件\n\n3. train_w2v_model.py\n 利用官方给的train, test训练词向量\n\n3. prepare_data.py\n 构建词表，词性表等\n\n4. load_data.py\n 加载数据\n\n5. model_dc.py\n 训练模型并预测\n" }, { "path": "code/TFNN/__init__.py", "content": "\n" }, { "path": "code/TFNN/activations.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\"\"\"\n 激活函数\n\"\"\"\nimport tensorflow as tf\n\n\ndef get_activation(activation=None):\n \"\"\"\n Get activation function accord to the parameter 'activation'\n Args:\n activation: str: 激活函数的名称\n Return:\n 激活函数\n \"\"\"\n if activation is None:\n return None\n elif activation == 'tanh':\n return tf.nn.tanh\n elif activation == 'relu':\n return tf.nn.relu\n elif activation == 'softmax':\n return tf.nn.softmax\n else:\n raise Exception('Unknow activation function: %s' % activation)\n" }, { "path": "code/TFNN/layers/ConvolutionalLayer.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\"\"\"\nDNN Layers:\n Convolutional1D\n\n\"\"\"\nimport numpy as np\nimport tensorflow as tf\n# from ..initializations import normal_weight\nfrom ..activations import get_activation\n\n\nclass Convolutional1D(object):\n\n def __init__(self, input_data, filter_length, nb_filter, strides=[1, 1, 1, 1],\n padding='VALID', activation='tanh', pooling=True,\n name='Convolutional1D'):\n \"\"\"1D卷积层\n Args:\n input_data: 3D tensor of shape=[batch_size, in_height, in_width]\n in_channels is set to 1 when use Convolutional1D.\n filter_length: int, 卷积核的长度，用于构造卷积核，在\n Convolutional1D中，卷积核shape=[filter_length, in_width, in_channels, nb_filters]\n nb_filter: int, 卷积核数量\n padding: 默认'VALID'，暂时不支持设成'SAME'\n pooling: bool, 是否池化\n \"\"\"\n assert padding in ('VALID'), 'Unknow padding %s' % padding\n # assert padding in ('VALID', 'SAME'), 'Unknow padding %s' % padding\n\n in_height, in_width = map(int, input_data.get_shape()[1:])\n self._input_data = tf.expand_dims(input_data, -1) # shape=[x, x, x, 1]\n self._filter_length = filter_length\n self._nb_filter = nb_filter\n self._strides = strides\n self._padding = padding\n self._activation = get_activation(activation)\n self._name = name\n self.pooling = pooling\n\n filter_length = self._filter_length\n nb_filter = self._nb_filter\n with tf.name_scope('%s_%d' % (name, filter_length)):\n if activation != 'relu':\n fan_in = filter_length * in_width\n fan_out = nb_filter * (in_width-filter_length+1)\n w_bound = np.sqrt(6. / (fan_in + fan_out))\n self.weights = tf.Variable(\n tf.random_uniform(\n minval=-w_bound, maxval=w_bound, dtype='float32',\n shape=[filter_length, in_width, 1, nb_filter]),\n name='conv_weight')\n tf.summary.histogram('weights', self.weights)\n else: # init weight for relu\n w_values = tf.random_normal(\n shape=[filter_length, in_width, 1, nb_filter]\n ) * tf.sqrt(2. / (filter_length * in_width * nb_filter))\n self.weights = tf.Variable(w_values, name='conv_weight')\n # bias\n self.biases = tf.Variable(\n tf.constant(0.1, shape=[nb_filter, ]),\n name='conv_bias')\n tf.summary.histogram('biases', self.biases)\n\n self.call()\n\n def call(self):\n # 卷积 if padding='VALID', then conv_output's shape=\n # [batch_size, in_height-filter_length+1, 1, nb_filters]\n conv_output = tf.nn.conv2d(\n input=self._input_data,\n filter=self.weights,\n strides=self._strides,\n padding=self._padding)\n\n # output's shape=[batch_size, new_height, 1, nb_filters]\n linear_output = tf.nn.bias_add(conv_output, self.biases)\n act_output = (\n linear_output if self._activation is None\n else self._activation(linear_output))\n if self.pooling:\n # max pooling, shape=[?, nb_filter]\n self._output = tf.reduce_max(tf.squeeze(act_output, [2]), 1)\n else:\n self._output = tf.squeeze(act_output, axis=2) # [?, n-w+1, nb_filter]\n\n @property\n def input_data(self):\n return self._input_data\n\n @property\n def output(self):\n return self._output\n\n @property\n def output_dim(self):\n return self._nb_filter\n\n def get_weights(self):\n return self.weights\n" }, { "path": "code/TFNN/layers/DenseLayer.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\"\"\"\nDNN Layers:\n SoftmaxLayer\n\n\"\"\"\nimport tensorflow as tf\nfrom ..activations import get_activation\n\n\nclass SoftmaxDense(object):\n\n def __init__(self, input_data, input_dim, output_dim, weights=None,\n biases=None, activation=None, name='Dense'):\n assert len(input_data.get_shape()) == 2, \\\n \"全连接层的输入必须要flatten, 即shape=[batch_size, input_dim]\"\n self._input_data = input_data\n self._input_dim = input_dim\n self._output_dim = output_dim\n self._activation = get_activation(activation)\n self._name = name\n\n with tf.name_scope(self._name):\n # initialize weights\n if weights is None:\n w_bound = tf.sqrt(6. / (input_dim + output_dim))\n weights = tf.Variable(\n tf.random_uniform(\n minval=-w_bound, maxval=w_bound, dtype='float32',\n shape=[input_dim, output_dim]),\n name='weights'\n )\n self._weights = weights\n tf.summary.histogram('weights', self._weights)\n # initialize biases\n if biases is None:\n biases = tf.Variable(\n tf.constant(0.1, shape=[self._output_dim]),\n name='biases')\n self._biases = biases\n tf.summary.histogram('biases', biases)\n\n self.call()\n\n def call(self):\n # output\n linear_output = tf.matmul(self._input_data, self._weights) + \\\n self._biases\n self._output = (\n linear_output if self._activation is None\n else self._activation(linear_output)\n )\n\n def loss(self, y):\n y = tf.cast(y, tf.int32)\n cross_entroy = tf.nn.sparse_softmax_cross_entropy_with_logits(\n logits=self.output, labels=y, name='xentroy')\n loss = tf.reduce_mean(cross_entroy, name='xentroy_mean')\n return loss\n\n def get_pre_y(self):\n # TODO 待修改\n # pre_y = tf.reshape(tf.round(tf.sigmoid(self._output)), [-1])\n pre_y = tf.arg_max(input=self._output, dimension=1)\n return pre_y\n\n @property\n def input_data(self):\n return self._input_data\n\n @property\n def input_dim(self):\n return self._input_dim\n\n @property\n def output_dim(self):\n return self._output_dim\n\n @property\n def name(self):\n return self._name\n\n @property\n def weights(self):\n return self._weights\n\n @property\n def biases(self):\n return self._biases\n\n @property\n def output(self):\n return self._output\n" }, { "path": "code/TFNN/layers/EmbeddingLayer.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\"\"\"\nDNN Layers:\n Embedding\n\n\"\"\"\nimport tensorflow as tf\n\n\nclass Embedding(object):\n\n def __init__(self, params, ids, name, keep_prob=1.0):\n with tf.name_scope('%s' % name):\n self._params = tf.Variable(params, tf.float32, name='embed')\n self._ids = ids\n\n # output\n embed_output = tf.nn.embedding_lookup(\n params=self._params,\n ids=self._ids\n )\n self._output = tf.nn.dropout(embed_output, keep_prob)\n\n @property\n def params(self):\n return self._params\n\n @property\n def output_dim(self):\n return int(self._output.get_shape()[-1])\n\n @property\n def output(self):\n return self._output" }, { "path": "code/TFNN/layers/__init__.py", "content": "\n" }, { "path": "code/TFNN/objectives.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\nimport tensorflow as tf\n\n\ndef categorical_crossentropy(y_true, y_pred):\n \"\"\"\n Args:\n y_true: int of list, length=batch_size\n y_pred: 2D tensor with shape=[batch_size, nb_classes]\n Returns:\n xx\n \"\"\"\n cross_entroy = tf.nn.sparse_softmax_cross_entropy_with_logits(\n logits=y_pred, labels=y_true, name='xentroy')\n return tf.reduce_mean(cross_entroy, name='xentroy_mean')\n" }, { "path": "code/TFNN/utils/__init__.py", "content": "\n" }, { "path": "code/TFNN/utils/data_util.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\nimport os\nimport pickle\nimport numpy as np\nfrom collections import defaultdict\n\n\ndef flatten_list(nest_list):\n \"\"\"\n 将嵌套列表压扁\n Args:\n nest_list: list,嵌套列表\n Return:\n flatten_list: list\n \"\"\"\n res = []\n for item in nest_list:\n if isinstance(item, list):\n res.extend(flatten_list(item))\n else:\n res.append(item)\n return res\n\n\ndef create_dictionary(items, dic_path, start=0, sort=True,\n min_count=None, lower=False, overwrite=False):\n \"\"\"\n 构建字典，并将构建的字典写入pkl文件中\n Args:\n items: list, [item_1, item_2, ...]\n dic_path: 需要保存的路径(以pkl结尾)\n start: int, voc起始下标，默认为0\n sort: bool, 是否按频率排序, 若为False，则按items排序\n min_count: 最小频次\n lower: bool, 是否转为小写\n overwrite: bool, 是否覆盖之前的文件\n Returns:\n None\n \"\"\"\n assert not dic_path.endswith('pk')\n if os.path.exists(dic_path) and not overwrite:\n return\n voc = dict()\n if sort:\n # 构建字典\n dic = defaultdict(int)\n for item in items:\n item = item if (not lower) else item.lower()\n dic[item] += 1\n # 排序\n dic = sorted(dic.items(), key=lambda d: d[1], reverse=True)\n for i, item in enumerate(dic):\n index = i + start\n key = item[0]\n if min_count and min_count > item[1]:\n continue\n voc[key] = index\n else: # 按items排序\n for i, item in enumerate(items):\n item = item if not lower else item.lower()\n index = i + start\n voc[item] = index\n # 写入文件\n file = open(dic_path, 'wb')\n pickle.dump(voc, file)\n file.close()\n\n\ndef map_item2id(items, voc, max_len, none_word=0, lower=False):\n \"\"\"\n 将word/pos等映射为id\n Args:\n items: list, 待映射列表\n voc: 词表\n max_len: int, 序列最大长度\n none_word: 未登录词标号,默认为0\n Returns:\n arr: np.array, dtype=int32, shape=[max_len,]\n \"\"\"\n assert type(none_word) == int\n arr = np.zeros((max_len,), dtype='int32')\n min_range = min(max_len, len(items))\n for i in range(min_range): # 若items长度大于max_len，则被截断\n item = items[i] if not lower else items[i].lower()\n arr[i] = voc[item] if item in voc else none_word\n return arr\n\n\ndef random_over_sampling():\n \"\"\"\n 随机过采样\n Args:\n xx\n Return:\n xx\n \"\"\"\n x_1 = [[1,1,1], [2,2,2], [3,3,3]]\n x_2 = [[1,1,1], [2,2,2], [3,3,3]]\n y = [1,2,3]\n from imblearn.over_samping import RandomOverSampler\n ros = RandomOverSampler(sandom_state=42)\n x_res, y_res = ros.fit_sample(x_1, y)\n print(x_res)\n print(y_res)\n\n\ndef demo():\n random_over_sampling()\n\n\nif __name__ == '__main__':\n demo()\n" }, { "path": "code/TFNN/utils/evaluate_util.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\nimport numpy as np\nfrom collections import defaultdict\nimport codecs\n\n\ndef sim_compute(pro_labels, right_labels, ignore_label=None):\n \"\"\"\n simple evaluate...\n Args:\n param pro_labels list : predict labels\n param right_labels list : right labels\n param ignore_label int : the label should be ignored\n Returns:\n pre, rec, f\n \"\"\"\n assert len(pro_labels) == len(right_labels)\n pre_pro_labels, pre_right_labels = [], []\n rec_pro_labels, rec_right_labels = [], []\n labels_len = len(pro_labels)\n for i in range(labels_len):\n pro_label = pro_labels[i]\n if pro_label != ignore_label: #\n pre_pro_labels.append(pro_label)\n pre_right_labels.append(right_labels[i])\n if right_labels[i] != ignore_label:\n rec_pro_labels.append(pro_label)\n rec_right_labels.append(right_labels[i])\n pre_pro_labels, pre_right_labels = np.array(pre_pro_labels, dtype='int32'), \\\n np.array(pre_right_labels, dtype='int32')\n rec_pro_labels, rec_right_labels = np.array(rec_pro_labels, dtype='int32'), \\\n np.array(rec_right_labels, dtype='int32')\n pre = 0. if len(pre_pro_labels) == 0 \\\n else len(np.where(pre_pro_labels == pre_right_labels)[0]) / float(len(pre_pro_labels))\n # rec = len(np.where(rec_pro_labels == rec_right_labels)[0]) / float(len(pre_pro_labels))\n rec = len(np.where(rec_pro_labels == rec_right_labels)[0]) / float(len(rec_right_labels))\n f = 0. if (pre + rec) == 0. \\\n else (pre * rec * 2.) / (pre + rec)\n return pre, rec, f\n\n\ndef demo():\n pro_labels = [1, 2, 3, 4, 0, 6, 7, 0, 2, 8]\n right_labels = [0, 2, 3, 6, 5, 4, 7, 1, 0, 3]\n # ignore_label = 0\n pre, rec, f = sim_compute(pro_labels, right_labels, ignore_label=2)\n print('pre:', pre)\n print('rec:', rec)\n print(' f:', f)\n\n\nif __name__ == '__main__':\n demo()\n" }, { "path": "code/TFNN/utils/io_util.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\nimport os\nimport codecs\n\n\ndef read_lines(path):\n lines = []\n with codecs.open(path, 'r', encoding='utf-8') as file:\n for line in file.readlines():\n line = line.rstrip()\n if line:\n lines.append(line)\n return lines\n\n\ndef get_file_list(path, postfix, file_list):\n \"\"\"\n 获取path路径下所有后缀为postfix的文件名\n Args:\n path str : 文件路径\n postfix str : 后缀\n file_list 存放文件路径\n Return:\n None\n \"\"\"\n temp_list = os.listdir(path)\n for fi in temp_list:\n fi_d = os.path.join(path, fi)\n if os.path.isdir(fi_d): # 若是目录，则递归\n get_file_list(fi_d, postfix, file_list)\n else: # 若是文件\n if fi_d.endswith(postfix): # 以postfix结尾\n file_list.append(fi_d)\n return None\n" }, { "path": "code/TFNN/utils/tensor_util.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\nimport tensorflow as tf\n\n\ndef zero_nil_slot(t, name=None):\n \"\"\"\n Overwrite the nil_slot (first 1 rows) of the input Tensor with zeros.\n Args:\n t: 2D tensor\n name: str\n Returns:\n Same shape as t\n \"\"\"\n with tf.name_scope('zero_nil_slot'):\n s = tf.shape(t)[1]\n z = tf.zeros([1, s], dtype=tf.float32)\n return tf.concat(\n axis=0, name=name,\n values=[z, tf.slice(t, [1, 0], [-1, -1])])\n\n\ndef add_gradient_noise(t, stddev=1e-3, name=None):\n \"\"\"\n Adds gradient noise as described in http://arxiv.org/abs/1511.06807 [2].\n The input Tensor `t` should be a gradient.\n The output will be `t` + gaussian noise.\n 0.001 was said to be a good fixed value for memory networks [2].\n Args:\n t: 2D tensor\n Returns:\n 2D tensor, same shape as t\n \"\"\"\n with tf.name_scope(\"add_gradient_noise\"):\n gn = tf.random_normal(tf.shape(t), stddev=stddev)\n return tf.add(t, gn, name=name)\n\n\ndef mask_tensor(input_data, lengths, maxlen, dtype=tf.float32):\n \"\"\"\n Args:\n input_data: 2D tensor\n lengths: integer vector, all its values < maxlen\n maxlen: scalar integer tensor\n dtype: str\n \"\"\"\n mask = tf.cast(tf.sequence_mask(lengths, maxlen), dtype)\n return tf.multiply(input_data, mask)\n" }, { "path": "code/configurations.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\"\"\"\n configurations\n\"\"\"\nimport os\n\n\n# --- corpus ---\nTRAIN_PATH = './Data/corpus/training.seg.csv'\nTEST_PATH = './Data/corpus/testing.seg.csv'\n\n\n# --- voc ---\nVOC_ROOT = './Data/voc'\nif not os.path.exists(VOC_ROOT):\n os.mkdir(VOC_ROOT)\nWORD_VOC_PATH = VOC_ROOT + '/word_voc.pkl'\nWORD_VOC_START = 2\nTAG_VOC_PATH = VOC_ROOT + '/tag_voc.pkl'\nTAG_VOC_START = 1\nLABEL_VOC_PATH = VOC_ROOT + '/label_voc.pkl'\n\n\n# --- embedding ---\nW2V_DIM = 256\nW2V_PATH = './Data/embedding/word2vec.pkl'\nEMBEDDING_ROOT = './Data/embedding/'\nif not os.path.exists(EMBEDDING_ROOT):\n os.mkdir(EMBEDDING_ROOT)\nW2V_TRAIN_PATH = EMBEDDING_ROOT + '/word2v.pkl'\nT2V_PATH = EMBEDDING_ROOT + '/tag2v.pkl'\nTAG_DIM = 64\n\n\n# --- training param ---\nMAX_LEN = 300\nBATCH_SIZE = 64\nNB_LABELS = 11\nNB_EPOCH = 30\nKEEP_PROB = 0.5\nWORD_KEEP_PROB = 0.9\nTAG_KEEP_PROB = 0.9\nKFOLD = 10\n" }, { "path": "code/load_data.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\"\"\"\n Load data.\n\"\"\"\nimport pickle\nfrom time import time\nimport numpy as np\nimport configurations as config\nfrom TFNN.utils.io_util import read_lines\nfrom TFNN.utils.data_util import map_item2id\n\n\ndef get_sentence_arr(words_tags, word_voc, tag_voc):\n \"\"\"\n 获取词序列\n Args:\n words_tags: list, 句子 and tags\n word_voc: 词表\n tag_voc: 词性标注表\n Returns:\n sentence_arr: np.array, 字符id序列\n tag_arr: np.array, 词性标记序列\n \"\"\"\n words, postags = [], []\n for item in words_tags:\n rindex = item.rindex('/')\n words.append(item[:rindex])\n postags.append(item[rindex+1:])\n # sentence arr\n sentence_arr = map_item2id(\n words, word_voc, config.MAX_LEN, lower=True)\n # pos tags arr\n postag_arr = map_item2id(\n postags, tag_voc, config.MAX_LEN, lower=False)\n return sentence_arr, postag_arr, len(words)\n\n\ndef init_data(lines, word_voc, tag_voc, label_voc):\n \"\"\"\n 加载数据\n Args:\n lines: list\n word_voc: dict, 词表\n tag_voc: dict, 词性标注表\n label_voc: dict\n Returns:\n sentences: np.array\n etc.\n \"\"\"\n data_count = len(lines)\n sentences = np.zeros((data_count, config.MAX_LEN), dtype='int32')\n tags = np.zeros((data_count, config.MAX_LEN), dtype='int32')\n sentence_actual_lengths = np.zeros((data_count,), dtype='int32')\n labels = np.zeros((data_count,), dtype='int32')\n instance_index = 0\n for i in range(data_count):\n index = lines[i].index(',')\n label = lines[i][:index]\n sentence = lines[i][index+1:]\n words_tags = sentence.split(' ')\n sentence_arr, tag_arr, actual_length = get_sentence_arr(words_tags, word_voc, tag_voc)\n\n sentences[instance_index, :] = sentence_arr\n tags[instance_index, :] = tag_arr\n sentence_actual_lengths[instance_index] = actual_length\n labels[instance_index] = label_voc[label] if label in label_voc else 0\n instance_index += 1\n return sentences, tags, labels\n\n\ndef load_embedding():\n \"\"\"\n 加载词向量、词性向量\n Return:\n word_weights: np.array\n tag_weights: np.array\n \"\"\"\n # 加载词向量\n with open(config.W2V_TRAIN_PATH, 'rb') as file_r:\n word_weights = pickle.load(file_r)\n # 加载tag向量\n with open(config.T2V_PATH, 'rb') as file_r:\n tag_weights = pickle.load(file_r)\n return word_weights, tag_weights\n\n\ndef load_voc():\n \"\"\"\n Load voc...\n Return:\n word_voc: dict\n tag_voc: dict\n label_voc: dict\n \"\"\"\n with open(config.WORD_VOC_PATH, 'rb') as file_r:\n word_voc = pickle.load(file_r)\n with open(config.TAG_VOC_PATH, 'rb') as file_r:\n tag_voc = pickle.load(file_r)\n with open(config.LABEL_VOC_PATH, 'rb') as file_r:\n label_voc = pickle.load(file_r)\n return word_voc, tag_voc, label_voc\n\n\ndef load_train_data(word_voc, tag_voc, label_voc):\n \"\"\"\n 加载训练测试数据\n Args:\n word_voc: dict\n tag_voc: dict\n label_voc: dict\n Returns:\n xx\n \"\"\"\n return init_data(read_lines(config.TRAIN_PATH), word_voc, tag_voc, label_voc)\n\n\ndef load_test_data(word_voc, tag_voc, label_voc):\n \"\"\"\n 加载测试数据\n Args:\n word_voc: dict\n tag_voc: dict\n label_voc: dict\n Returns:\n xx\n \"\"\"\n sentences, tags, _ = init_data(read_lines(config.TEST_PATH), word_voc, tag_voc, label_voc)\n return sentences, tags\n\n\ndef demo():\n t0 = time()\n word_weights, tag_weights = load_embedding()\n word_voc, tag_voc, label_voc = load_voc()\n data, label_voc = load_train_data()\n sentences, tags, labels = data[:]\n print(sentences.shape)\n print(tags.shape)\n print(labels.shape)\n print(word_weights.shape)\n print(tag_weights.shape)\n print('Done in %ds!' % (time()-t0))\n\n\nif __name__ == '__main__':\n demo()\n" }, { "path": "code/model_dc.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\nimport os\nfrom time import time\nimport configurations as config\nimport tensorflow as tf\nimport numpy as np\nfrom load_data import load_embedding, load_voc, load_train_data, load_test_data\nfrom TFNN.layers.EmbeddingLayer import Embedding\nfrom TFNN.layers.DenseLayer import SoftmaxDense\nfrom TFNN.layers.ConvolutionalLayer import Convolutional1D\nfrom TFNN.utils.evaluate_util import sim_compute\nfrom TFNN.utils.tensor_util import zero_nil_slot\nfrom sklearn.model_selection import KFold\nimport codecs\nfrom TFNN.utils.io_util import read_lines\n\n\nclass DCModel(object):\n\n def __init__(self, max_len, word_weights, tag_weights, result_path=None, label_voc=None):\n \"\"\"\n Initilize model\n Args:\n max_len: int, 句子最大长度\n word_weights: np.array, shape=[|V_words|, w2v_dim]，词向量\n tag_weights: np.array, shape=[|V_tags|, t2v_dim],标记向量\n result_path: str, 模型评价结果存放路径\n label_voc: dict\n \"\"\"\n self._result_path = result_path\n self._label_voc = label_voc\n self._label_voc_rev = dict()\n for key in self._label_voc:\n value = self._label_voc[key]\n self._label_voc_rev[value] = key\n\n # input placeholders\n self.input_sentence_ph = tf.placeholder(\n tf.int32, shape=(None, max_len), name='input_sentence_ph')\n self.input_tag_ph = tf.placeholder(tf.int32, shape=(None, max_len), name='input_tag_ph')\n self.label_ph = tf.placeholder(tf.int32, shape=(None,), name='label_ph')\n self.keep_prob_ph = tf.placeholder(tf.float32, name='keep_prob')\n self.word_keep_prob_ph = tf.placeholder(tf.float32, name='word_keep_prob')\n self.tag_keep_prob_ph = tf.placeholder(tf.float32, name='tag_keep_prob')\n\n # embedding layers\n self.nil_vars = set()\n word_embed_layer = Embedding(\n params=word_weights, ids=self.input_sentence_ph,\n keep_prob=self.word_keep_prob_ph, name='word_embed_layer')\n tag_embed_layer = Embedding(\n params=tag_weights, ids=self.input_tag_ph,\n keep_prob=self.tag_keep_prob_ph, name='tag_embed_layer')\n self.nil_vars.add(word_embed_layer.params.name)\n self.nil_vars.add(tag_embed_layer.params.name)\n\n # sentence representation\n sentence_input = tf.concat(\n values=[word_embed_layer.output, tag_embed_layer.output], axis=2)\n\n # sentence conv\n conv_layer = Convolutional1D(\n input_data=sentence_input, filter_length=3,\n nb_filter=1000, activation='relu', name='conv_layer')\n\n # dense layer\n dense_input_drop = tf.nn.dropout(conv_layer.output, self.keep_prob_ph)\n self.dense_layer = SoftmaxDense(\n input_data=dense_input_drop, input_dim=conv_layer.output_dim,\n output_dim=config.NB_LABELS, name='output_layer')\n\n self.loss = self.dense_layer.loss(self.label_ph) + \\\n 0.001*tf.nn.l2_loss(self.dense_layer.weights)\n optimizer = tf.train.AdamOptimizer() # Adam\n grads_and_vars = optimizer.compute_gradients(self.loss)\n nil_grads_and_vars = []\n for g, v in grads_and_vars:\n if v.name in self.nil_vars:\n nil_grads_and_vars.append((zero_nil_slot(g), v))\n else:\n nil_grads_and_vars.append((g, v))\n global_step = tf.Variable(0, name='global_step', trainable=False)\n\n # train op\n self.train_op = optimizer.apply_gradients(\n nil_grads_and_vars, name='train_op', global_step=global_step)\n\n # pre op\n self.pre_op = self.dense_layer.get_pre_y()\n\n # summary\n gpu_options = tf.GPUOptions(visible_device_list='0', allow_growth=True)\n self.sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))\n\n # init model\n init = tf.global_variables_initializer()\n self.sess.run(init)\n\n def fit(self, sentences_train, tags_train, labels_train,\n sentences_dev=None, tags_dev=None, labels_dev=None,\n sentences_test=None, tags_test=None, labels_test=None,\n batch_size=64, nb_epoch=40, keep_prob=1.0, word_keep_prob=1.0,\n tag_keep_prob=1.0, seed=137):\n \"\"\"\n fit model\n Args:\n sentences_train, tags_train, labels_train: 训练数据\n sentences_dev, tags_dev, labels_dev: 开发数据\n batch_size: int, batch size\n nb_epoch: int, 迭代次数\n keep_prob: float between [0, 1], 全连接层前的dropout\n word_keep_prob: float between [0, 1], 词向量层dropout\n tag_keep_prob: float between [0, 1], 标记向量层dropout\n \"\"\"\n self.nb_epoch_scores = [] # 存放nb_epoch次迭代的f值\n nb_train = int(labels_train.shape[0] / batch_size) + 1\n for step in range(nb_epoch):\n print('Epoch %d / %d:' % (step+1, nb_epoch))\n # shuffle\n np.random.seed(seed)\n np.random.shuffle(sentences_train)\n np.random.seed(seed)\n np.random.shuffle(tags_train)\n np.random.seed(seed)\n np.random.shuffle(labels_train)\n\n # train\n total_loss = 0.\n for i in range(nb_train):\n # for i in range(nb_train):\n sentences_feed = sentences_train[i*batch_size:(i+1)*batch_size]\n tags_feed = tags_train[i*batch_size:(i+1)*batch_size]\n labels_feed = labels_train[i*batch_size:(i+1)*batch_size]\n feed_dict = {\n self.input_sentence_ph: sentences_feed,\n self.input_tag_ph: tags_feed,\n self.label_ph: labels_feed,\n self.keep_prob_ph: keep_prob,\n self.word_keep_prob_ph: word_keep_prob,\n self.tag_keep_prob_ph: tag_keep_prob,\n }\n _, loss_value = self.sess.run(\n [self.train_op, self.loss], feed_dict=feed_dict)\n total_loss += loss_value\n\n total_loss /= float(nb_train)\n\n # 计算在训练集、开发集、测试集上的性能\n p_train, r_train, f_train = self.evaluate(sentences_train, tags_train, labels_train)\n p_dev, r_dev, f_dev = self.evaluate(sentences_dev, tags_dev, labels_dev)\n pre_labels = self.predict(sentences_test, tags_test)\n with codecs.open('./Data/result/epoch_%d.csv' % (step+1), 'w', encoding='utf-8') as file_w:\n for num, label in enumerate(pre_labels):\n file_w.write('%d,%s\\n' % (num+1, self._label_voc_rev[label]))\n self.nb_epoch_scores.append([p_dev, r_dev, f_dev])\n print('\\tloss=%f, train f=%f, dev f=%f' % (total_loss, f_train, f_dev))\n\n def predict(self, data_sentences, data_tags, batch_size=50):\n \"\"\"\n Args:\n data_sentences, data_tags: np.array\n batch_size: int\n Return:\n pre_labels: list\n \"\"\"\n pre_labels = []\n nb_test = int(data_sentences.shape[0]/batch_size) + 1\n for i in range(nb_test):\n sentences_feed = data_sentences[i*batch_size:(i+1)*batch_size]\n tags_feed = data_tags[i*batch_size:(i+1)*batch_size]\n feed_dict = {\n self.input_sentence_ph: sentences_feed,\n self.input_tag_ph: tags_feed,\n self.keep_prob_ph: 1.0,\n self.word_keep_prob_ph: 1.0,\n self.tag_keep_prob_ph: 1.0}\n pre_temp = self.sess.run(self.pre_op, feed_dict=feed_dict)\n pre_labels += list(pre_temp)\n return pre_labels\n\n def evaluate(self, data_sentences, data_tags, data_labels,\n ignore_label=None, batch_size=64, simple_compute=True):\n \"\"\"\n Args:\n data_sentences, data_tags, data_labels: np.array\n ignore_label: int, 负例的编号，或者None\n simple_compute: bool, 是否画出性能详细指标表格\n Return:\n p, r, f1\n \"\"\"\n pre_labels = []\n nb_dev = int(len(data_labels)/batch_size) + 1\n for i in range(nb_dev):\n sentences_feed = data_sentences[i*batch_size:(i+1)*batch_size]\n tags_feed = data_tags[i*batch_size:(i+1)*batch_size]\n labels_feed = data_labels[i*batch_size:(i+1)*batch_size]\n feed_dict = {\n self.input_sentence_ph: sentences_feed,\n self.input_tag_ph: tags_feed,\n self.label_ph: labels_feed,\n self.keep_prob_ph: 1.0,\n self.word_keep_prob_ph: 1.0,\n self.tag_keep_prob_ph: 1.0}\n pre_temp = self.sess.run(self.pre_op, feed_dict=feed_dict)\n pre_labels += list(pre_temp)\n right_labels = data_labels[:len(pre_labels)]\n pre, rec, f = sim_compute(pre_labels, right_labels, ignore_label=ignore_label)\n return pre, rec, f\n\n def clear_model(self):\n tf.reset_default_graph() #\n self.sess.close()\n\n def get_best_score(self):\n \"\"\"\n 计算模型得分(当开发集上f值达到最高时所对应的测试集得分)\n Returns:\n score: float, 开发集达到最高时,测试集的[p, r, f]\n nb_epoch: int, the num of epoch\n \"\"\"\n # nb_epoch_scores = sorted(self.nb_epoch_scores, key=lambda d: d[1][-1], reverse=True)\n nb_epoch, best_score = -1, None\n for i in range(len(self.nb_epoch_scores)):\n if not best_score or self.nb_epoch_scores[i][-1] > best_score[-1]:\n best_score = self.nb_epoch_scores[i]\n nb_epoch = i\n return best_score, nb_epoch\n\n\ndef predict():\n word_weights, tag_weights = load_embedding()\n word_voc, tag_voc, label_voc = load_voc()\n\n # train data\n sentences, tags, labels = load_train_data(word_voc, tag_voc, label_voc)\n seed = 137\n np.random.seed(seed)\n np.random.shuffle(sentences)\n np.random.seed(seed)\n np.random.shuffle(tags)\n np.random.seed(seed)\n np.random.shuffle(labels)\n\n # load data\n sentences_test, tags_test = load_test_data(word_voc, tag_voc, label_voc)\n labels_test = None\n \n # clear reslut\n if not os.path.exists('./Data/result'):\n os.mkdir('./Data/result')\n command = 'rm ./Data/result/*'\n os.popen(command)\n\n # 划分训练、开发、测试集\n kf = KFold(n_splits=config.KFOLD)\n train_indices, dev_indices = [], []\n for train_index, dev_index in kf.split(labels):\n train_indices.append(train_index)\n dev_indices.append(dev_index)\n for num in range(config.KFOLD):\n train_index, dev_index = train_indices[num], dev_indices[num]\n sentences_train, sentences_dev = sentences[train_index], sentences[dev_index]\n tags_train, tags_dev = tags[train_index], tags[dev_index]\n labels_train, labels_dev = labels[train_index], labels[dev_index]\n\n # init model\n model = DCModel(\n config.MAX_LEN, word_weights, tag_weights, result_path='./Data/result/result.txt',\n label_voc=label_voc)\n\n # fit model\n model.fit(\n sentences_train, tags_train, labels_train,\n sentences_dev, tags_dev, labels_dev,\n sentences_test, tags_test, labels_test,\n config.BATCH_SIZE, config.NB_EPOCH, keep_prob=config.KEEP_PROB,\n word_keep_prob=config.WORD_KEEP_PROB, tag_keep_prob=config.TAG_KEEP_PROB)\n print(model.get_best_score())\n [p_test, r_test, f_test], nb_epoch = model.get_best_score()\n command = 'cp ./Data/result/epoch_%d.csv ./Data/result/best_%d' % (nb_epoch+1, num)\n print(command)\n os.popen(command)\n print(p_test, r_test, f_test, '\\n')\n # evaluate\n # result_path_k = result_path % k\n # p_test, r_test, f_test = model.evaluate(sentences_test, tags_test, positions_test,\n # labels_test, simple_compute=False, ignore_label=IGNORE_LABEL,\n # label_voc=relation_voc, result_path=result_path_k)\n # clear model\n model.clear_model()\n del model\n\n\ndef init_result():\n labels = []\n for i in range(config.KFOLD):\n lines = read_lines('./Data/result/best_%d' % i)\n temp = []\n for line in lines:\n label = line.split(',')[1]\n temp.append(label)\n labels.append(temp)\n return labels\n\n\ndef merge():\n datas = init_result()\n data_count = len(datas[0])\n label_type_count = config.NB_LABELS\n labels = np.zeros((data_count, label_type_count))\n for data in datas:\n for i, label in enumerate(data):\n label_id = int(label) - 1\n labels[i][label_id] += 1\n # 取众数\n final_labels = []\n for item in labels:\n label = item.argmax() + 1\n final_labels.append(label)\n\n # clear result\n command = 'rm ./Data/result/*'\n os.popen(command)\n\n with codecs.open('./Data/result/integrade.csv', 'w', encoding='utf-8') as file_w:\n for i, label in enumerate(final_labels):\n file_w.write('%d,%d\\n' % (i+1, label))\n print('Result: %s' % file_w.name)\n\n\n\nif __name__ == '__main__':\n t0 = time()\n\n # predict test data\n predict()\n\n # merge\n merge()\n\n print('Done in %ds!' % (time()-t0))\n" }, { "path": "code/prepare_data.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\"\"\"\n prepare data.\n\n 生成:\n word voc\n position voc\n relation type voc\n\n lookup tables\n\"\"\"\nimport os\nimport pickle\nimport numpy as np\nimport configurations as config\nfrom TFNN.utils.data_util import create_dictionary\nfrom TFNN.utils.io_util import read_lines\nfrom time import time\n\n\ndef init_voc():\n \"\"\"\n 初始化voc\n \"\"\"\n lines = read_lines(config.TRAIN_PATH)\n lines += read_lines(config.TEST_PATH)\n words = [] # 句子\n pos_tags = [] # 词性标记类型\n for line in lines:\n index = line.index(',')\n sentence = line[index+1:]\n # words and tags\n words_tags = sentence.split(' ')\n words_temp, tag_temp = [], []\n for item in words_tags:\n r_index = item.rindex('/')\n word, tag = item[:r_index], item[r_index+1:]\n words_temp.append(word)\n tag_temp.append(tag)\n pos_tags.extend(tag_temp)\n words.extend(words_temp)\n # word voc\n create_dictionary(\n words, config.WORD_VOC_PATH, start=config.WORD_VOC_START,\n min_count=5, sort=True, lower=True, overwrite=True)\n # tag voc\n create_dictionary(\n pos_tags, config.TAG_VOC_PATH, start=config.TAG_VOC_START,\n sort=True, lower=False, overwrite=True)\n # label voc\n label_types = [str(i) for i in range(1, 12)]\n create_dictionary(\n label_types, config.LABEL_VOC_PATH, start=0, overwrite=True)\n\n\ndef init_word_embedding(path=None, overwrite=False):\n \"\"\"\n 初始化word embedding\n Args:\n path: 结果存放路径\n \"\"\"\n if os.path.exists(path) and not overwrite:\n return\n with open(config.W2V_PATH, 'rb') as file:\n w2v_dict_full = pickle.load(file)\n with open(config.WORD_VOC_PATH, 'rb') as file:\n w2id_dict = pickle.load(file)\n word_voc_size = len(w2id_dict.keys()) + config.WORD_VOC_START\n word_weights = np.zeros((word_voc_size, config.W2V_DIM), dtype='float32')\n for word in w2id_dict:\n index = w2id_dict[word] # 词的标号\n if word in w2v_dict_full:\n word_weights[index, :] = w2v_dict_full[word]\n else:\n random_vec = np.random.uniform(\n -0.25, 0.25, size=(config.W2V_DIM,)).astype('float32')\n word_weights[index, :] = random_vec\n # 写入pkl文件\n with open(path, 'wb') as file:\n pickle.dump(word_weights, file, protocol=2)\n\n\ndef init_tag_embedding(path, overwrite=False):\n \"\"\"\n 初始化pos tag embedding\n Args:\n path: 结果存放路径\n \"\"\"\n if os.path.exists(path) and not overwrite:\n return\n with open(config.TAG_VOC_PATH, 'rb') as file:\n tag_voc = pickle.load(file)\n tag_voc_size = len(tag_voc.keys()) + config.TAG_VOC_START\n tag_weights = np.random.normal(\n size=(tag_voc_size, config.TAG_DIM)).astype('float32')\n for i in range(config.TAG_VOC_START):\n tag_weights[i, :] = 0.\n with open(path, 'wb') as file:\n pickle.dump(tag_weights, file, protocol=2)\n\n\ndef init_embedding():\n \"\"\"\n 初始化embedding\n \"\"\"\n if not os.path.exists(config.EMBEDDING_ROOT):\n os.mkdir(config.EMBEDDING_ROOT)\n # 初始化word embedding\n init_word_embedding(config.W2V_TRAIN_PATH, overwrite=True)\n # 初始化tag embedding\n init_tag_embedding(config.T2V_PATH, overwrite=True)\n\n\ndef demo():\n with open(config.W2V_TRAIN_PATH, 'rb') as file:\n temp = pickle.load(file)\n print(temp.shape)\n\n\nif __name__ == '__main__':\n t0 = time()\n\n init_voc() # 初始化voc\n\n init_embedding() # 初始化embedding\n\n demo()\n\n print('Done in %.1fs!' % (time()-t0))\n" }, { "path": "code/train_w2v_model.py", "content": "#!/usr/bin/env python\n# coding=utf-8\nimport codecs\nimport pickle\nfrom gensim.models import Word2Vec\nfrom gensim.models.word2vec import LineSentence\nfrom gensim.models.keyedvectors import KeyedVectors\nfrom TFNN.utils.io_util import read_lines\n\n\ndef get_sentence(sentence_tag):\n words = []\n for item in sentence_tag.split(' '):\n index = item.rindex('/')\n words.append(item[:index])\n return ' '.join(words)\n\n\ndef extract_sentece():\n lines = read_lines('./Data/corpus/training.seg.csv')\n lines += read_lines('./Data/corpus/testing.seg.csv')\n with codecs.open('./Data/corpus/sentence.txt', 'w', encoding='utf-8') as file_w:\n for line in lines:\n index = line.index(',')\n word_tag = line[index+1:]\n file_w.write('%s\\n' % get_sentence(word_tag))\n\n\ndef train():\n extract_sentece()\n\n in_path = './Data/corpus/sentence.txt'\n out_path = './Data/embedding/word2vec.bin'\n # 训练模型\n model = Word2Vec(\n sg=1, sentences=LineSentence(in_path),\n size=256, window=5, min_count=3, workers=4, iter=40)\n model.wv.save_word2vec_format(out_path, binary=True)\n\n\ndef bin2pkl():\n model = KeyedVectors.load_word2vec_format('./Data/embedding/word2vec.bin', binary=True)\n word_dict = {}\n for word in model.vocab:\n word_dict[word] = model[word]\n with open('./Data/embedding/word2vec.pkl', 'wb') as file_w:\n pickle.dump(word_dict, file_w)\n print(file_w.name)\n\n\n\nif __name__ == '__main__':\n train()\n\n bin2pkl()\n" } ]