Full Code of liu-nlper/DocumentClassification for AI

Repository: liu-nlper/DocumentClassification
Branch: master
Commit: b6c6cce3638a
Files: 20
Total size: 38.2 KB

Directory structure:
gitextract__8lu9u9y/

├── README.md
└── code/
    ├── Data/
    │   └── corpus/
    │       └── corpus download link
    ├── README
    ├── TFNN/
    │   ├── __init__.py
    │   ├── activations.py
    │   ├── layers/
    │   │   ├── ConvolutionalLayer.py
    │   │   ├── DenseLayer.py
    │   │   ├── EmbeddingLayer.py
    │   │   └── __init__.py
    │   ├── objectives.py
    │   └── utils/
    │       ├── __init__.py
    │       ├── data_util.py
    │       ├── evaluate_util.py
    │       ├── io_util.py
    │       └── tensor_util.py
    ├── configurations.py
    ├── load_data.py
    ├── model_dc.py
    ├── prepare_data.py
    └── train_w2v_model.py

================================================
FILE CONTENTS
================================================

================================================
FILE: README.md
================================================
# Document Classification
This code implements a simple CNN model for document classification with tensorflow.

# Model Structure
![model.png](https://github.com/MRliujiaxin/DocumentClassification/raw/master/model.png)

# Requirements
- Python: 2.7
- Tensorflow: 1.0.0
- Numpy: 1.12.1
- sklearn: 0.18.1
- gensim: 1.0.1
- pickle


================================================
FILE: code/Data/corpus/corpus download link
================================================
数据下载地址：http://competition.ai100.com.cn/html/game_det.html?id=24

将分词/词性标注后的文件命名为training.seg.csv和testing.seg.csv，放置到当前目录下。

处理后的文本如下所示:
  "公司/n 是/vshi 经/p 批准/v 依法/d 从事/vi 融资/vi 性/ng 担保/vn 业务/n 的/ude1 ..."


================================================
FILE: code/README
================================================
下载数据到Data/corpus目录下，并做分词和词性标注处理，再按以下顺序执行:
    train_w2v_model.py -> prepare_data.py -> model_dc.py


文件说明:

1. 分词、词性标注采用中科院NLPIR，处理后文件:
    ./Data/corpus/training.seg.csv
    ./Data/corpus/testing.seg.csv

2. configuration.py
    配置文件

3. train_w2v_model.py
    利用官方给的train, test训练词向量

3. prepare_data.py
    构建词表，词性表等

4. load_data.py
    加载数据

5. model_dc.py
    训练模型并预测


================================================
FILE: code/TFNN/__init__.py
================================================



================================================
FILE: code/TFNN/activations.py
================================================
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    激活函数
"""
import tensorflow as tf


def get_activation(activation=None):
    """
    Get activation function accord to the parameter 'activation'
    Args:
        activation: str: 激活函数的名称
    Return:
        激活函数
    """
    if activation is None:
        return None
    elif activation == 'tanh':
        return tf.nn.tanh
    elif activation == 'relu':
        return tf.nn.relu
    elif activation == 'softmax':
        return tf.nn.softmax
    else:
        raise Exception('Unknow activation function: %s' % activation)


================================================
FILE: code/TFNN/layers/ConvolutionalLayer.py
================================================
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
DNN Layers:
    Convolutional1D

"""
import numpy as np
import tensorflow as tf
# from ..initializations import normal_weight
from ..activations import get_activation


class Convolutional1D(object):

    def __init__(self, input_data, filter_length, nb_filter, strides=[1, 1, 1, 1],
                 padding='VALID', activation='tanh', pooling=True,
                 name='Convolutional1D'):
        """1D卷积层
        Args:
            input_data: 3D tensor of shape=[batch_size, in_height, in_width]
                in_channels is set to 1 when use Convolutional1D.
            filter_length: int, 卷积核的长度，用于构造卷积核，在
                Convolutional1D中，卷积核shape=[filter_length, in_width, in_channels, nb_filters]
            nb_filter: int, 卷积核数量
            padding: 默认'VALID'，暂时不支持设成'SAME'
            pooling: bool, 是否池化
        """
        assert padding in ('VALID'), 'Unknow padding %s' % padding
        # assert padding in ('VALID', 'SAME'), 'Unknow padding %s' % padding

        in_height, in_width = map(int, input_data.get_shape()[1:])
        self._input_data = tf.expand_dims(input_data, -1)  # shape=[x, x, x, 1]
        self._filter_length = filter_length
        self._nb_filter = nb_filter
        self._strides = strides
        self._padding = padding
        self._activation = get_activation(activation)
        self._name = name
        self.pooling = pooling

        filter_length = self._filter_length
        nb_filter = self._nb_filter
        with tf.name_scope('%s_%d' % (name, filter_length)):
            if activation != 'relu':
                fan_in = filter_length * in_width
                fan_out = nb_filter * (in_width-filter_length+1)
                w_bound = np.sqrt(6. / (fan_in + fan_out))
                self.weights = tf.Variable(
                    tf.random_uniform(
                        minval=-w_bound, maxval=w_bound, dtype='float32',
                        shape=[filter_length, in_width, 1, nb_filter]),
                    name='conv_weight')
                tf.summary.histogram('weights', self.weights)
            else:  # init weight for relu
                w_values = tf.random_normal(
                    shape=[filter_length, in_width, 1, nb_filter]
                ) * tf.sqrt(2. / (filter_length * in_width * nb_filter))
                self.weights = tf.Variable(w_values, name='conv_weight')
            # bias
            self.biases = tf.Variable(
                tf.constant(0.1, shape=[nb_filter, ]),
                name='conv_bias')
            tf.summary.histogram('biases', self.biases)

        self.call()

    def call(self):
        # 卷积  if padding='VALID', then conv_output's shape=
        #   [batch_size, in_height-filter_length+1, 1, nb_filters]
        conv_output = tf.nn.conv2d(
            input=self._input_data,
            filter=self.weights,
            strides=self._strides,
            padding=self._padding)

        # output's shape=[batch_size, new_height, 1, nb_filters]
        linear_output = tf.nn.bias_add(conv_output, self.biases)
        act_output = (
            linear_output if self._activation is None
            else self._activation(linear_output))
        if self.pooling:
            # max pooling, shape=[?, nb_filter]
            self._output = tf.reduce_max(tf.squeeze(act_output, [2]), 1)
        else:
            self._output = tf.squeeze(act_output, axis=2)  # [?, n-w+1, nb_filter]

    @property
    def input_data(self):
        return self._input_data

    @property
    def output(self):
        return self._output

    @property
    def output_dim(self):
        return self._nb_filter

    def get_weights(self):
        return self.weights


================================================
FILE: code/TFNN/layers/DenseLayer.py
================================================
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
DNN Layers:
    SoftmaxLayer

"""
import tensorflow as tf
from ..activations import get_activation


class SoftmaxDense(object):

    def __init__(self, input_data, input_dim, output_dim, weights=None,
                 biases=None, activation=None, name='Dense'):
        assert len(input_data.get_shape()) == 2, \
            "全连接层的输入必须要flatten, 即shape=[batch_size, input_dim]"
        self._input_data = input_data
        self._input_dim = input_dim
        self._output_dim = output_dim
        self._activation = get_activation(activation)
        self._name = name

        with tf.name_scope(self._name):
            # initialize weights
            if weights is None:
                w_bound = tf.sqrt(6. / (input_dim + output_dim))
                weights = tf.Variable(
                    tf.random_uniform(
                        minval=-w_bound, maxval=w_bound, dtype='float32',
                        shape=[input_dim, output_dim]),
                    name='weights'
                )
            self._weights = weights
            tf.summary.histogram('weights', self._weights)
            # initialize biases
            if biases is None:
                biases = tf.Variable(
                    tf.constant(0.1, shape=[self._output_dim]),
                    name='biases')
            self._biases = biases
            tf.summary.histogram('biases', biases)

        self.call()

    def call(self):
        # output
        linear_output = tf.matmul(self._input_data, self._weights) + \
                            self._biases
        self._output = (
            linear_output if self._activation is None
            else self._activation(linear_output)
        )

    def loss(self, y):
        y = tf.cast(y, tf.int32)
        cross_entroy = tf.nn.sparse_softmax_cross_entropy_with_logits(
            logits=self.output, labels=y, name='xentroy')
        loss = tf.reduce_mean(cross_entroy, name='xentroy_mean')
        return loss

    def get_pre_y(self):
        # TODO 待修改
        # pre_y = tf.reshape(tf.round(tf.sigmoid(self._output)), [-1])
        pre_y = tf.arg_max(input=self._output, dimension=1)
        return pre_y

    @property
    def input_data(self):
        return self._input_data

    @property
    def input_dim(self):
        return self._input_dim

    @property
    def output_dim(self):
        return self._output_dim

    @property
    def name(self):
        return self._name

    @property
    def weights(self):
        return self._weights

    @property
    def biases(self):
        return self._biases

    @property
    def output(self):
        return self._output


================================================
FILE: code/TFNN/layers/EmbeddingLayer.py
================================================
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
DNN Layers:
    Embedding

"""
import tensorflow as tf


class Embedding(object):

    def __init__(self, params, ids, name, keep_prob=1.0):
        with tf.name_scope('%s' % name):
            self._params = tf.Variable(params, tf.float32, name='embed')
            self._ids = ids

            # output
            embed_output = tf.nn.embedding_lookup(
                params=self._params,
                ids=self._ids
            )
            self._output = tf.nn.dropout(embed_output, keep_prob)

    @property
    def params(self):
        return self._params

    @property
    def output_dim(self):
        return int(self._output.get_shape()[-1])

    @property
    def output(self):
        return self._output

================================================
FILE: code/TFNN/layers/__init__.py
================================================



================================================
FILE: code/TFNN/objectives.py
================================================
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import tensorflow as tf


def categorical_crossentropy(y_true, y_pred):
    """
    Args:
        y_true: int of list, length=batch_size
        y_pred: 2D tensor with shape=[batch_size, nb_classes]
    Returns:
        xx
    """
    cross_entroy = tf.nn.sparse_softmax_cross_entropy_with_logits(
        logits=y_pred, labels=y_true, name='xentroy')
    return tf.reduce_mean(cross_entroy, name='xentroy_mean')


================================================
FILE: code/TFNN/utils/__init__.py
================================================



================================================
FILE: code/TFNN/utils/data_util.py
================================================
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import pickle
import numpy as np
from collections import defaultdict


def flatten_list(nest_list):
    """
    将嵌套列表压扁
    Args:
        nest_list: list,嵌套列表
    Return:
        flatten_list: list
    """
    res = []
    for item in nest_list:
        if isinstance(item, list):
            res.extend(flatten_list(item))
        else:
            res.append(item)
    return res


def create_dictionary(items, dic_path, start=0, sort=True,
                      min_count=None, lower=False, overwrite=False):
    """
    构建字典，并将构建的字典写入pkl文件中
    Args:
        items: list, [item_1, item_2, ...]
        dic_path: 需要保存的路径(以pkl结尾)
        start: int, voc起始下标，默认为0
        sort: bool, 是否按频率排序, 若为False，则按items排序
        min_count: 最小频次
        lower: bool, 是否转为小写
        overwrite: bool, 是否覆盖之前的文件
    Returns:
        None
    """
    assert not dic_path.endswith('pk')
    if os.path.exists(dic_path) and not overwrite:
        return
    voc = dict()
    if sort:
        # 构建字典
        dic = defaultdict(int)
        for item in items:
            item = item if (not lower) else item.lower()
            dic[item] += 1
        # 排序
        dic = sorted(dic.items(), key=lambda d: d[1], reverse=True)
        for i, item in enumerate(dic):
            index = i + start
            key = item[0]
            if min_count and min_count > item[1]:
                continue
            voc[key] = index
    else:  # 按items排序
        for i, item in enumerate(items):
            item = item if not lower else item.lower()
            index = i + start
            voc[item] = index
    # 写入文件
    file = open(dic_path, 'wb')
    pickle.dump(voc, file)
    file.close()


def map_item2id(items, voc, max_len, none_word=0, lower=False):
    """
    将word/pos等映射为id
    Args:
        items: list, 待映射列表
        voc: 词表
        max_len: int, 序列最大长度
        none_word: 未登录词标号,默认为0
    Returns:
        arr: np.array, dtype=int32, shape=[max_len,]
    """
    assert type(none_word) == int
    arr = np.zeros((max_len,), dtype='int32')
    min_range = min(max_len, len(items))
    for i in range(min_range):  # 若items长度大于max_len，则被截断
        item = items[i] if not lower else items[i].lower()
        arr[i] = voc[item] if item in voc else none_word
    return arr


def random_over_sampling():
    """
    随机过采样
    Args:
        xx
    Return:
        xx
    """
    x_1 = [[1,1,1], [2,2,2], [3,3,3]]
    x_2 = [[1,1,1], [2,2,2], [3,3,3]]
    y = [1,2,3]
    from imblearn.over_samping import RandomOverSampler
    ros = RandomOverSampler(sandom_state=42)
    x_res, y_res = ros.fit_sample(x_1, y)
    print(x_res)
    print(y_res)


def demo():
    random_over_sampling()


if __name__ == '__main__':
    demo()


================================================
FILE: code/TFNN/utils/evaluate_util.py
================================================
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import numpy as np
from collections import defaultdict
import codecs


def sim_compute(pro_labels, right_labels, ignore_label=None):
    """
    simple evaluate...
    Args:
        param pro_labels list : predict labels
        param right_labels list : right labels
        param ignore_label int : the label should be ignored
    Returns:
        pre, rec, f
    """
    assert len(pro_labels) == len(right_labels)
    pre_pro_labels, pre_right_labels = [], []
    rec_pro_labels, rec_right_labels = [], []
    labels_len = len(pro_labels)
    for i in range(labels_len):
        pro_label = pro_labels[i]
        if pro_label != ignore_label:  #
            pre_pro_labels.append(pro_label)
            pre_right_labels.append(right_labels[i])
        if right_labels[i] != ignore_label:
            rec_pro_labels.append(pro_label)
            rec_right_labels.append(right_labels[i])
    pre_pro_labels, pre_right_labels = np.array(pre_pro_labels, dtype='int32'), \
        np.array(pre_right_labels, dtype='int32')
    rec_pro_labels, rec_right_labels = np.array(rec_pro_labels, dtype='int32'), \
        np.array(rec_right_labels, dtype='int32')
    pre = 0. if len(pre_pro_labels) == 0 \
        else len(np.where(pre_pro_labels == pre_right_labels)[0]) / float(len(pre_pro_labels))
    # rec = len(np.where(rec_pro_labels == rec_right_labels)[0]) / float(len(pre_pro_labels))
    rec = len(np.where(rec_pro_labels == rec_right_labels)[0]) / float(len(rec_right_labels))
    f = 0. if (pre + rec) == 0. \
        else (pre * rec * 2.) / (pre + rec)
    return pre, rec, f


def demo():
    pro_labels = [1, 2, 3, 4, 0, 6, 7, 0, 2, 8]
    right_labels = [0, 2, 3, 6, 5, 4, 7, 1, 0, 3]
    # ignore_label = 0
    pre, rec, f = sim_compute(pro_labels, right_labels, ignore_label=2)
    print('pre:', pre)
    print('rec:', rec)
    print('  f:', f)


if __name__ == '__main__':
    demo()


================================================
FILE: code/TFNN/utils/io_util.py
================================================
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import codecs


def read_lines(path):
    lines = []
    with codecs.open(path, 'r', encoding='utf-8') as file:
        for line in file.readlines():
            line = line.rstrip()
            if line:
                lines.append(line)
    return lines


def get_file_list(path, postfix, file_list):
    """
    获取path路径下所有后缀为postfix的文件名
    Args:
        path str : 文件路径
        postfix str : 后缀
        file_list 存放文件路径
    Return:
        None
    """
    temp_list = os.listdir(path)
    for fi in temp_list:
        fi_d = os.path.join(path, fi)
        if os.path.isdir(fi_d):  # 若是目录，则递归
            get_file_list(fi_d, postfix, file_list)
        else:  # 若是文件
            if fi_d.endswith(postfix):  # 以postfix结尾
                file_list.append(fi_d)
    return None


================================================
FILE: code/TFNN/utils/tensor_util.py
================================================
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import tensorflow as tf


def zero_nil_slot(t, name=None):
    """
    Overwrite the nil_slot (first 1 rows) of the input Tensor with zeros.
    Args:
        t: 2D tensor
        name: str
    Returns:
        Same shape as t
    """
    with tf.name_scope('zero_nil_slot'):
        s = tf.shape(t)[1]
        z = tf.zeros([1, s], dtype=tf.float32)
        return tf.concat(
            axis=0, name=name,
            values=[z, tf.slice(t, [1, 0], [-1, -1])])


def add_gradient_noise(t, stddev=1e-3, name=None):
    """
    Adds gradient noise as described in http://arxiv.org/abs/1511.06807 [2].
    The input Tensor `t` should be a gradient.
    The output will be `t` + gaussian noise.
    0.001 was said to be a good fixed value for memory networks [2].
    Args:
        t: 2D tensor
    Returns:
        2D tensor, same shape as t
    """
    with tf.name_scope("add_gradient_noise"):
        gn = tf.random_normal(tf.shape(t), stddev=stddev)
        return tf.add(t, gn, name=name)


def mask_tensor(input_data, lengths, maxlen, dtype=tf.float32):
    """
    Args:
        input_data: 2D tensor
        lengths: integer vector, all its values < maxlen
        maxlen: scalar integer tensor
        dtype: str
    """
    mask = tf.cast(tf.sequence_mask(lengths, maxlen), dtype)
    return tf.multiply(input_data, mask)


================================================
FILE: code/configurations.py
================================================
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    configurations
"""
import os


# --- corpus ---
TRAIN_PATH = './Data/corpus/training.seg.csv'
TEST_PATH = './Data/corpus/testing.seg.csv'


# --- voc ---
VOC_ROOT = './Data/voc'
if not os.path.exists(VOC_ROOT):
    os.mkdir(VOC_ROOT)
WORD_VOC_PATH = VOC_ROOT + '/word_voc.pkl'
WORD_VOC_START = 2
TAG_VOC_PATH = VOC_ROOT + '/tag_voc.pkl'
TAG_VOC_START = 1
LABEL_VOC_PATH = VOC_ROOT + '/label_voc.pkl'


# --- embedding ---
W2V_DIM = 256
W2V_PATH = './Data/embedding/word2vec.pkl'
EMBEDDING_ROOT = './Data/embedding/'
if not os.path.exists(EMBEDDING_ROOT):
    os.mkdir(EMBEDDING_ROOT)
W2V_TRAIN_PATH = EMBEDDING_ROOT + '/word2v.pkl'
T2V_PATH = EMBEDDING_ROOT + '/tag2v.pkl'
TAG_DIM = 64


# --- training param ---
MAX_LEN = 300
BATCH_SIZE = 64
NB_LABELS = 11
NB_EPOCH = 30
KEEP_PROB = 0.5
WORD_KEEP_PROB = 0.9
TAG_KEEP_PROB = 0.9
KFOLD = 10


================================================
FILE: code/load_data.py
================================================
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    Load data.
"""
import pickle
from time import time
import numpy as np
import configurations as config
from TFNN.utils.io_util import read_lines
from TFNN.utils.data_util import map_item2id


def get_sentence_arr(words_tags, word_voc, tag_voc):
    """
    获取词序列
    Args:
        words_tags: list, 句子 and tags
        word_voc: 词表
        tag_voc: 词性标注表
    Returns:
        sentence_arr: np.array, 字符id序列
        tag_arr: np.array, 词性标记序列
    """
    words, postags = [], []
    for item in words_tags:
        rindex = item.rindex('/')
        words.append(item[:rindex])
        postags.append(item[rindex+1:])
    # sentence arr
    sentence_arr = map_item2id(
        words, word_voc, config.MAX_LEN, lower=True)
    # pos tags arr
    postag_arr = map_item2id(
        postags, tag_voc, config.MAX_LEN, lower=False)
    return sentence_arr, postag_arr, len(words)


def init_data(lines, word_voc, tag_voc, label_voc):
    """
    加载数据
    Args:
        lines: list
        word_voc: dict, 词表
        tag_voc: dict, 词性标注表
        label_voc: dict
    Returns:
        sentences: np.array
        etc.
    """
    data_count = len(lines)
    sentences = np.zeros((data_count, config.MAX_LEN), dtype='int32')
    tags = np.zeros((data_count, config.MAX_LEN), dtype='int32')
    sentence_actual_lengths = np.zeros((data_count,), dtype='int32')
    labels = np.zeros((data_count,), dtype='int32')
    instance_index = 0
    for i in range(data_count):
        index = lines[i].index(',')
        label = lines[i][:index]
        sentence = lines[i][index+1:]
        words_tags = sentence.split(' ')
        sentence_arr, tag_arr, actual_length = get_sentence_arr(words_tags, word_voc, tag_voc)

        sentences[instance_index, :] = sentence_arr
        tags[instance_index, :] = tag_arr
        sentence_actual_lengths[instance_index] = actual_length
        labels[instance_index] = label_voc[label] if label in label_voc else 0
        instance_index += 1
    return sentences, tags, labels


def load_embedding():
    """
    加载词向量、词性向量
    Return:
        word_weights: np.array
        tag_weights: np.array
    """
    # 加载词向量
    with open(config.W2V_TRAIN_PATH, 'rb') as file_r:
        word_weights = pickle.load(file_r)
    # 加载tag向量
    with open(config.T2V_PATH, 'rb') as file_r:
        tag_weights = pickle.load(file_r)
    return word_weights, tag_weights


def load_voc():
    """
    Load voc...
    Return:
        word_voc: dict
        tag_voc: dict
        label_voc: dict
    """
    with open(config.WORD_VOC_PATH, 'rb') as file_r:
        word_voc = pickle.load(file_r)
    with open(config.TAG_VOC_PATH, 'rb') as file_r:
        tag_voc = pickle.load(file_r)
    with open(config.LABEL_VOC_PATH, 'rb') as file_r:
        label_voc = pickle.load(file_r)
    return word_voc, tag_voc, label_voc


def load_train_data(word_voc, tag_voc, label_voc):
    """
    加载训练测试数据
    Args:
        word_voc: dict
        tag_voc: dict
        label_voc: dict
    Returns:
        xx
    """
    return init_data(read_lines(config.TRAIN_PATH), word_voc, tag_voc, label_voc)


def load_test_data(word_voc, tag_voc, label_voc):
    """
    加载测试数据
    Args:
        word_voc: dict
        tag_voc: dict
        label_voc: dict
    Returns:
        xx
    """
    sentences, tags, _ = init_data(read_lines(config.TEST_PATH), word_voc, tag_voc, label_voc)
    return sentences, tags


def demo():
    t0 = time()
    word_weights, tag_weights = load_embedding()
    word_voc, tag_voc, label_voc = load_voc()
    data, label_voc = load_train_data()
    sentences, tags, labels = data[:]
    print(sentences.shape)
    print(tags.shape)
    print(labels.shape)
    print(word_weights.shape)
    print(tag_weights.shape)
    print('Done in %ds!' % (time()-t0))


if __name__ == '__main__':
    demo()


================================================
FILE: code/model_dc.py
================================================
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
from time import time
import configurations as config
import tensorflow as tf
import numpy as np
from load_data import load_embedding, load_voc, load_train_data, load_test_data
from TFNN.layers.EmbeddingLayer import Embedding
from TFNN.layers.DenseLayer import SoftmaxDense
from TFNN.layers.ConvolutionalLayer import Convolutional1D
from TFNN.utils.evaluate_util import sim_compute
from TFNN.utils.tensor_util import zero_nil_slot
from sklearn.model_selection import KFold
import codecs
from TFNN.utils.io_util import read_lines


class DCModel(object):

    def __init__(self, max_len, word_weights, tag_weights, result_path=None, label_voc=None):
        """
        Initilize model
        Args:
            max_len: int, 句子最大长度
            word_weights: np.array, shape=[|V_words|, w2v_dim]，词向量
            tag_weights: np.array, shape=[|V_tags|, t2v_dim],标记向量
            result_path: str, 模型评价结果存放路径
            label_voc: dict
        """
        self._result_path = result_path
        self._label_voc = label_voc
        self._label_voc_rev = dict()
        for key in self._label_voc:
            value = self._label_voc[key]
            self._label_voc_rev[value] = key

        # input placeholders
        self.input_sentence_ph = tf.placeholder(
            tf.int32, shape=(None, max_len), name='input_sentence_ph')
        self.input_tag_ph = tf.placeholder(tf.int32, shape=(None, max_len), name='input_tag_ph')
        self.label_ph = tf.placeholder(tf.int32, shape=(None,), name='label_ph')
        self.keep_prob_ph = tf.placeholder(tf.float32, name='keep_prob')
        self.word_keep_prob_ph = tf.placeholder(tf.float32, name='word_keep_prob')
        self.tag_keep_prob_ph = tf.placeholder(tf.float32, name='tag_keep_prob')

        # embedding layers
        self.nil_vars = set()
        word_embed_layer = Embedding(
            params=word_weights, ids=self.input_sentence_ph,
            keep_prob=self.word_keep_prob_ph, name='word_embed_layer')
        tag_embed_layer = Embedding(
            params=tag_weights, ids=self.input_tag_ph,
            keep_prob=self.tag_keep_prob_ph, name='tag_embed_layer')
        self.nil_vars.add(word_embed_layer.params.name)
        self.nil_vars.add(tag_embed_layer.params.name)

        # sentence representation
        sentence_input = tf.concat(
            values=[word_embed_layer.output, tag_embed_layer.output], axis=2)

        # sentence conv
        conv_layer = Convolutional1D(
            input_data=sentence_input, filter_length=3,
            nb_filter=1000, activation='relu', name='conv_layer')

        # dense layer
        dense_input_drop = tf.nn.dropout(conv_layer.output, self.keep_prob_ph)
        self.dense_layer = SoftmaxDense(
            input_data=dense_input_drop, input_dim=conv_layer.output_dim,
            output_dim=config.NB_LABELS, name='output_layer')

        self.loss = self.dense_layer.loss(self.label_ph) + \
            0.001*tf.nn.l2_loss(self.dense_layer.weights)
        optimizer = tf.train.AdamOptimizer()  # Adam
        grads_and_vars = optimizer.compute_gradients(self.loss)
        nil_grads_and_vars = []
        for g, v in grads_and_vars:
            if v.name in self.nil_vars:
                nil_grads_and_vars.append((zero_nil_slot(g), v))
            else:
                nil_grads_and_vars.append((g, v))
        global_step = tf.Variable(0, name='global_step', trainable=False)

        # train op
        self.train_op = optimizer.apply_gradients(
            nil_grads_and_vars, name='train_op', global_step=global_step)

        # pre op
        self.pre_op = self.dense_layer.get_pre_y()

        # summary
        gpu_options = tf.GPUOptions(visible_device_list='0', allow_growth=True)
        self.sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))

        # init model
        init = tf.global_variables_initializer()
        self.sess.run(init)

    def fit(self, sentences_train, tags_train, labels_train,
            sentences_dev=None, tags_dev=None, labels_dev=None,
            sentences_test=None, tags_test=None, labels_test=None,
            batch_size=64, nb_epoch=40, keep_prob=1.0, word_keep_prob=1.0,
            tag_keep_prob=1.0, seed=137):
        """
        fit model
        Args:
            sentences_train, tags_train, labels_train: 训练数据
            sentences_dev, tags_dev, labels_dev: 开发数据
            batch_size: int, batch size
            nb_epoch: int, 迭代次数
            keep_prob: float between [0, 1], 全连接层前的dropout
            word_keep_prob: float between [0, 1], 词向量层dropout
            tag_keep_prob: float between [0, 1], 标记向量层dropout
        """
        self.nb_epoch_scores = []  # 存放nb_epoch次迭代的f值
        nb_train = int(labels_train.shape[0] / batch_size) + 1
        for step in range(nb_epoch):
            print('Epoch %d / %d:' % (step+1, nb_epoch))
            # shuffle
            np.random.seed(seed)
            np.random.shuffle(sentences_train)
            np.random.seed(seed)
            np.random.shuffle(tags_train)
            np.random.seed(seed)
            np.random.shuffle(labels_train)

            # train
            total_loss = 0.
            for i in range(nb_train):
                # for i in range(nb_train):
                sentences_feed = sentences_train[i*batch_size:(i+1)*batch_size]
                tags_feed = tags_train[i*batch_size:(i+1)*batch_size]
                labels_feed = labels_train[i*batch_size:(i+1)*batch_size]
                feed_dict = {
                    self.input_sentence_ph: sentences_feed,
                    self.input_tag_ph: tags_feed,
                    self.label_ph: labels_feed,
                    self.keep_prob_ph: keep_prob,
                    self.word_keep_prob_ph: word_keep_prob,
                    self.tag_keep_prob_ph: tag_keep_prob,
                }
                _, loss_value = self.sess.run(
                    [self.train_op, self.loss], feed_dict=feed_dict)
                total_loss += loss_value

            total_loss /= float(nb_train)

            #  计算在训练集、开发集、测试集上的性能
            p_train, r_train, f_train = self.evaluate(sentences_train, tags_train, labels_train)
            p_dev, r_dev, f_dev = self.evaluate(sentences_dev, tags_dev, labels_dev)
            pre_labels = self.predict(sentences_test, tags_test)
            with codecs.open('./Data/result/epoch_%d.csv' % (step+1), 'w', encoding='utf-8') as file_w:
                for num, label in enumerate(pre_labels):
                    file_w.write('%d,%s\n' % (num+1, self._label_voc_rev[label]))
            self.nb_epoch_scores.append([p_dev, r_dev, f_dev])
            print('\tloss=%f, train f=%f, dev f=%f' % (total_loss, f_train, f_dev))

    def predict(self, data_sentences, data_tags, batch_size=50):
        """
        Args:
            data_sentences, data_tags: np.array
            batch_size: int
        Return:
            pre_labels: list
        """
        pre_labels = []
        nb_test = int(data_sentences.shape[0]/batch_size) + 1
        for i in range(nb_test):
            sentences_feed = data_sentences[i*batch_size:(i+1)*batch_size]
            tags_feed = data_tags[i*batch_size:(i+1)*batch_size]
            feed_dict = {
                self.input_sentence_ph: sentences_feed,
                self.input_tag_ph: tags_feed,
                self.keep_prob_ph: 1.0,
                self.word_keep_prob_ph: 1.0,
                self.tag_keep_prob_ph: 1.0}
            pre_temp = self.sess.run(self.pre_op, feed_dict=feed_dict)
            pre_labels += list(pre_temp)
        return pre_labels

    def evaluate(self, data_sentences, data_tags, data_labels,
                 ignore_label=None, batch_size=64, simple_compute=True):
        """
        Args:
            data_sentences, data_tags, data_labels: np.array
            ignore_label: int, 负例的编号，或者None
            simple_compute: bool, 是否画出性能详细指标表格
        Return:
            p, r, f1
        """
        pre_labels = []
        nb_dev = int(len(data_labels)/batch_size) + 1
        for i in range(nb_dev):
            sentences_feed = data_sentences[i*batch_size:(i+1)*batch_size]
            tags_feed = data_tags[i*batch_size:(i+1)*batch_size]
            labels_feed = data_labels[i*batch_size:(i+1)*batch_size]
            feed_dict = {
                self.input_sentence_ph: sentences_feed,
                self.input_tag_ph: tags_feed,
                self.label_ph: labels_feed,
                self.keep_prob_ph: 1.0,
                self.word_keep_prob_ph: 1.0,
                self.tag_keep_prob_ph: 1.0}
            pre_temp = self.sess.run(self.pre_op, feed_dict=feed_dict)
            pre_labels += list(pre_temp)
        right_labels = data_labels[:len(pre_labels)]
        pre, rec, f = sim_compute(pre_labels, right_labels, ignore_label=ignore_label)
        return pre, rec, f

    def clear_model(self):
        tf.reset_default_graph()  #
        self.sess.close()

    def get_best_score(self):
        """
        计算模型得分(当开发集上f值达到最高时所对应的测试集得分)
        Returns:
            score: float, 开发集达到最高时,测试集的[p, r, f]
            nb_epoch: int, the num of epoch
        """
        # nb_epoch_scores = sorted(self.nb_epoch_scores, key=lambda d: d[1][-1], reverse=True)
        nb_epoch, best_score = -1, None
        for i in range(len(self.nb_epoch_scores)):
            if not best_score or self.nb_epoch_scores[i][-1] > best_score[-1]:
                best_score = self.nb_epoch_scores[i]
                nb_epoch = i
        return best_score, nb_epoch


def predict():
    word_weights, tag_weights = load_embedding()
    word_voc, tag_voc, label_voc = load_voc()

    # train data
    sentences, tags, labels = load_train_data(word_voc, tag_voc, label_voc)
    seed = 137
    np.random.seed(seed)
    np.random.shuffle(sentences)
    np.random.seed(seed)
    np.random.shuffle(tags)
    np.random.seed(seed)
    np.random.shuffle(labels)

    # load data
    sentences_test, tags_test = load_test_data(word_voc, tag_voc, label_voc)
    labels_test = None
    
    # clear reslut
    if not os.path.exists('./Data/result'):
        os.mkdir('./Data/result')
    command = 'rm ./Data/result/*'
    os.popen(command)

    # 划分训练、开发、测试集
    kf = KFold(n_splits=config.KFOLD)
    train_indices, dev_indices = [], []
    for train_index, dev_index in kf.split(labels):
        train_indices.append(train_index)
        dev_indices.append(dev_index)
    for num in range(config.KFOLD):
        train_index, dev_index = train_indices[num], dev_indices[num]
        sentences_train, sentences_dev = sentences[train_index], sentences[dev_index]
        tags_train, tags_dev = tags[train_index], tags[dev_index]
        labels_train, labels_dev = labels[train_index], labels[dev_index]

        # init model
        model = DCModel(
            config.MAX_LEN, word_weights, tag_weights, result_path='./Data/result/result.txt',
            label_voc=label_voc)

        # fit model
        model.fit(
            sentences_train, tags_train, labels_train,
            sentences_dev, tags_dev, labels_dev,
            sentences_test, tags_test, labels_test,
            config.BATCH_SIZE, config.NB_EPOCH, keep_prob=config.KEEP_PROB,
            word_keep_prob=config.WORD_KEEP_PROB, tag_keep_prob=config.TAG_KEEP_PROB)
        print(model.get_best_score())
        [p_test, r_test, f_test], nb_epoch = model.get_best_score()
        command = 'cp ./Data/result/epoch_%d.csv ./Data/result/best_%d' % (nb_epoch+1, num)
        print(command)
        os.popen(command)
        print(p_test, r_test, f_test, '\n')
        # evaluate
        # result_path_k = result_path % k
        # p_test, r_test, f_test = model.evaluate(sentences_test, tags_test, positions_test,
        #    labels_test, simple_compute=False, ignore_label=IGNORE_LABEL,
        #    label_voc=relation_voc, result_path=result_path_k)
        # clear model
        model.clear_model()
        del model


def init_result():
    labels = []
    for i in range(config.KFOLD):
        lines = read_lines('./Data/result/best_%d' % i)
        temp = []
        for line in lines:
            label = line.split(',')[1]
            temp.append(label)
        labels.append(temp)
    return labels


def merge():
    datas = init_result()
    data_count = len(datas[0])
    label_type_count = config.NB_LABELS
    labels = np.zeros((data_count, label_type_count))
    for data in datas:
        for i, label in enumerate(data):
            label_id = int(label) - 1
            labels[i][label_id] += 1
    # 取众数
    final_labels = []
    for item in labels:
        label = item.argmax() + 1
        final_labels.append(label)

    # clear result
    command = 'rm ./Data/result/*'
    os.popen(command)

    with codecs.open('./Data/result/integrade.csv', 'w', encoding='utf-8') as file_w:
        for i, label in enumerate(final_labels):
            file_w.write('%d,%d\n' % (i+1, label))
        print('Result: %s' % file_w.name)



if __name__ == '__main__':
    t0 = time()

    # predict test data
    predict()

    # merge
    merge()

    print('Done in %ds!' % (time()-t0))


================================================
FILE: code/prepare_data.py
================================================
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    prepare data.

    生成:
        word voc
        position voc
        relation type voc

        lookup tables
"""
import os
import pickle
import numpy as np
import configurations as config
from TFNN.utils.data_util import create_dictionary
from TFNN.utils.io_util import read_lines
from time import time


def init_voc():
    """
    初始化voc
    """
    lines = read_lines(config.TRAIN_PATH)
    lines += read_lines(config.TEST_PATH)
    words = []  # 句子
    pos_tags = []  # 词性标记类型
    for line in lines:
        index = line.index(',')
        sentence = line[index+1:]
        # words and tags
        words_tags = sentence.split(' ')
        words_temp, tag_temp = [], []
        for item in words_tags:
            r_index = item.rindex('/')
            word, tag = item[:r_index], item[r_index+1:]
            words_temp.append(word)
            tag_temp.append(tag)
        pos_tags.extend(tag_temp)
        words.extend(words_temp)
    # word voc
    create_dictionary(
        words, config.WORD_VOC_PATH, start=config.WORD_VOC_START,
        min_count=5, sort=True, lower=True, overwrite=True)
    # tag voc
    create_dictionary(
        pos_tags, config.TAG_VOC_PATH, start=config.TAG_VOC_START,
        sort=True, lower=False, overwrite=True)
    # label voc
    label_types = [str(i) for i in range(1, 12)]
    create_dictionary(
        label_types, config.LABEL_VOC_PATH, start=0, overwrite=True)


def init_word_embedding(path=None, overwrite=False):
    """
    初始化word embedding
    Args:
        path: 结果存放路径
    """
    if os.path.exists(path) and not overwrite:
        return
    with open(config.W2V_PATH, 'rb') as file:
        w2v_dict_full = pickle.load(file)
    with open(config.WORD_VOC_PATH, 'rb') as file:
        w2id_dict = pickle.load(file)
    word_voc_size = len(w2id_dict.keys()) + config.WORD_VOC_START
    word_weights = np.zeros((word_voc_size, config.W2V_DIM), dtype='float32')
    for word in w2id_dict:
        index = w2id_dict[word]  # 词的标号
        if word in w2v_dict_full:
            word_weights[index, :] = w2v_dict_full[word]
        else:
            random_vec = np.random.uniform(
                -0.25, 0.25, size=(config.W2V_DIM,)).astype('float32')
            word_weights[index, :] = random_vec
    # 写入pkl文件
    with open(path, 'wb') as file:
        pickle.dump(word_weights, file, protocol=2)


def init_tag_embedding(path, overwrite=False):
    """
    初始化pos tag embedding
    Args:
        path: 结果存放路径
    """
    if os.path.exists(path) and not overwrite:
        return
    with open(config.TAG_VOC_PATH, 'rb') as file:
        tag_voc = pickle.load(file)
    tag_voc_size = len(tag_voc.keys()) + config.TAG_VOC_START
    tag_weights = np.random.normal(
        size=(tag_voc_size, config.TAG_DIM)).astype('float32')
    for i in range(config.TAG_VOC_START):
        tag_weights[i, :] = 0.
    with open(path, 'wb') as file:
        pickle.dump(tag_weights, file, protocol=2)


def init_embedding():
    """
    初始化embedding
    """
    if not os.path.exists(config.EMBEDDING_ROOT):
        os.mkdir(config.EMBEDDING_ROOT)
    # 初始化word embedding
    init_word_embedding(config.W2V_TRAIN_PATH, overwrite=True)
    # 初始化tag embedding
    init_tag_embedding(config.T2V_PATH, overwrite=True)


def demo():
    with open(config.W2V_TRAIN_PATH, 'rb') as file:
        temp = pickle.load(file)
    print(temp.shape)


if __name__ == '__main__':
    t0 = time()

    init_voc()  # 初始化voc

    init_embedding()  # 初始化embedding

    demo()

    print('Done in %.1fs!' % (time()-t0))


================================================
FILE: code/train_w2v_model.py
================================================
#!/usr/bin/env python
# coding=utf-8
import codecs
import pickle
from gensim.models import Word2Vec
from gensim.models.word2vec import LineSentence
from gensim.models.keyedvectors import KeyedVectors
from TFNN.utils.io_util import read_lines


def get_sentence(sentence_tag):
    words = []
    for item in sentence_tag.split(' '):
        index = item.rindex('/')
        words.append(item[:index])
    return ' '.join(words)


def extract_sentece():
    lines = read_lines('./Data/corpus/training.seg.csv')
    lines += read_lines('./Data/corpus/testing.seg.csv')
    with codecs.open('./Data/corpus/sentence.txt', 'w', encoding='utf-8') as file_w:
        for line in lines:
            index = line.index(',')
            word_tag = line[index+1:]
            file_w.write('%s\n' % get_sentence(word_tag))


def train():
    extract_sentece()

    in_path = './Data/corpus/sentence.txt'
    out_path = './Data/embedding/word2vec.bin'
    # 训练模型
    model = Word2Vec(
        sg=1, sentences=LineSentence(in_path),
        size=256, window=5, min_count=3, workers=4, iter=40)
    model.wv.save_word2vec_format(out_path, binary=True)


def bin2pkl():
    model = KeyedVectors.load_word2vec_format('./Data/embedding/word2vec.bin', binary=True)
    word_dict = {}
    for word in model.vocab:
        word_dict[word] = model[word]
    with open('./Data/embedding/word2vec.pkl', 'wb') as file_w:
        pickle.dump(word_dict, file_w)
        print(file_w.name)



if __name__ == '__main__':
    train()

    bin2pkl()