Full Code of hrwhisper/SpamMessage for AI

Repository: hrwhisper/SpamMessage
Branch: master
Commit: 754d3a74c626
Files: 23
Total size: 80.6 MB

Directory structure:
gitextract_y8xnqj4p/

├── .gitignore
├── classifier/
│   ├── LogisticRegression.py
│   ├── NaiveBayesian.py
│   ├── Perceptron.py
│   └── __init__.py
├── data/
│   ├── 不带标签短信.txt
│   └── 带标签短信.txt
├── judgeSpamMessage.py
├── model/
│   ├── Bayes_sklearn.pkl
│   ├── LogisticRegression.pkl
│   ├── Logistic_sklearn.pkl
│   ├── NaiveBayesian.pkl
│   ├── Perceptron.pkl
│   ├── SVM_sklearn.pkl
│   ├── __init__.py
│   └── vsm.pkl
├── model_manage.py
├── readme.md
├── test.py
├── test_jieba.py
├── test_judge.py
├── test_judge2.py
└── token_and_save_to_file.py

================================================
FILE CONTENTS
================================================

================================================
FILE: .gitignore
================================================
.idea
*.pyc
*~
*.swp

================================================
FILE: classifier/LogisticRegression.py
================================================
# -*- coding: utf-8 -*-
# @Date    : 2016/10/16
# @Author  : hrwhisper
import random
import sklearn
import numpy as np
from sklearn.utils.extmath import safe_sparse_dot


class LogisticRegression(sklearn.base.BaseEstimator):
    def __init__(self, alpha=0.1, max_iter=100):  # 0.02 200 0.9927425
        self.alpha = alpha
        self.max_iter = max_iter
        self.weights = None

    def _sigmoid(self, x):
        return .5 * (1 + np.tanh(.5 * x))  # 1.0 / (1 + np.exp(-x))

    def fit(self, X, y):
        """
        :param X: sparse matrix(n_samples,n_features) the training feature vector
        :param y:  array-like(n_samples,) Target vector relative to X.
        :return: self
        """
        m, n = X.shape
        target = np.array(y).reshape((m, 1))
        self.weights = np.ones((n, 1))
        for _ in range(self.max_iter):
            h = self._sigmoid(safe_sparse_dot(X, self.weights))  # X * weights
            error = target - h
            self.weights += self.alpha * safe_sparse_dot(X.T, error)  # alpha * X.T * error
        return self

    def predict(self, X):
        return (self._sigmoid(safe_sparse_dot(X, self.weights)) > 0.5).ravel().astype('int')


if __name__ == '__main__':
    a = [1, 2, 3]
    b = [4, 5, 6]
    a = np.array(a)
    b = np.array(b)
    print(a, b)
    a = a.reshape((-1, 1))
    print(a)
    print(a - b)


================================================
FILE: classifier/NaiveBayesian.py
================================================
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2016/11/29 22:04
# @Author  : wqs
# @File    : NaiveBayesian.py

import sklearn
import numpy as np
from scipy.sparse import csr_matrix
from sklearn.utils.extmath import safe_sparse_dot
from sklearn.preprocessing import LabelBinarizer


class NaiveBayesian(sklearn.base.BaseEstimator):
    def __init__(self, alpha=1.0):
        self.alpha = alpha
        self.classes = None
        self.class_log_prior = None
        self.feature_log_prob = None

    def fit(self, X, y):
        _, n = X.shape
        labelbin = LabelBinarizer()
        Y = labelbin.fit_transform(y)
        self.classes = labelbin.classes_
        Y = np.concatenate((1 - Y, Y), axis=1).astype(np.float64)

        class_count = np.zeros(2, dtype=np.float64)
        feature_count = np.zeros((2, n), dtype=np.float64)

        feature_count += safe_sparse_dot(Y.T, X)  # count frequency by y.T * X
        class_count += Y.sum(axis=0)

        smoothed_fc = feature_count + self.alpha
        smoothed_cc = smoothed_fc.sum(axis=1)
        self.feature_log_prob = (np.log(smoothed_fc) - np.log(smoothed_cc.reshape(-1, 1)))
        # self.class_log_prior = np.zeros(2) - np.log(2)
        self.class_log_prior = np.log(class_count / sum(class_count))
        return self

    def predict(self, X):
        jll = safe_sparse_dot(X, self.feature_log_prob.T) + self.class_log_prior
        return self.classes[np.argmax(jll, axis=1)]


================================================
FILE: classifier/Perceptron.py
================================================
# -*- coding: utf-8 -*-
# @Date    : 2016/11/1
# @Author  : hrwhisper

import numpy as np
import sklearn
from sklearn.utils.extmath import safe_sparse_dot


class Perceptron(sklearn.base.BaseEstimator):
    def __init__(self, alpha=0.1, max_iter=100):
        self.threshold = 0.5
        self.alpha = alpha
        self.max_iter = max_iter
        self.weights = None

    def fit(self, X, y):
        """
            :param X: sparse matrix(n_samples,n_features) the training feature vector
            :param y:  array-like(n_samples,) Target vector relative to X.
            :return: self
        """
        m, n = X.shape
        target = np.array(y).reshape((m, 1))
        self.weights = np.ones((n, 1))
        for _ in range(self.max_iter):
            h = safe_sparse_dot(X, self.weights) > self.threshold
            error = target - h
            self.weights += self.alpha * safe_sparse_dot(X.T, error)
        return self

    def predict(self, X):
        return (safe_sparse_dot(X, self.weights) > self.threshold).ravel().astype('int')


================================================
FILE: classifier/__init__.py
================================================
# -*- coding: utf-8 -*-
# @Date    : 2016/10/16
# @Author  : hrwhisper

================================================
FILE: data/不带标签短信.txt
================================================
[File too large to display: 13.5 MB]

================================================
FILE: data/带标签短信.txt
================================================
[File too large to display: 55.5 MB]

================================================
FILE: judgeSpamMessage.py
================================================
# -*- coding: utf-8 -*-
# @Date    : 2016/12/3
# @Author  : hrwhisper
import codecs
import sys
from collections import Counter
from optparse import OptionParser
from multiprocessing import Pool
import jieba
from sklearn.externals import joblib
from model_manage import BowTransform
import time


def token(x):
    return Counter(jieba.lcut(x))


if __name__ == "__main__":
    parser = OptionParser()
    parser.add_option('-c', '--classifier', dest="cls_name", type='string', default='p',
                      help="define the classifier you want to use:  \t\t\n"
                           "p  => Perceptron,\t\t\t\t\t\n"
                           "lr => LogisticRegression,\t\t\t\t\t\n"
                           "nb => NaiveBayesian,\t\t\t\t\t\n"
                           "svm => SVM(sklearn),\t\t\t\t\t\n"
                           "lrs => LogisticRegression(sklearn),\t\t\t\t\t\n"
                           "nbs => NaiveBayesian(sklearn),")

    parser.add_option('-i', '--input', dest="input_filename", type='string', default='./data/不带标签短信.txt',
                      help="input file name")

    parser.add_option('-o', '--output', dest="output_filename", type='string', default='./data/result.txt',
                      help="output file name")

    options, args = parser.parse_args()
    #
    classifiers = {
        'p': './model/Perceptron.pkl',  # 0.1 2000
        'lr': './model/LogisticRegression.pkl',  # 0.2 2000
        'nb': './model/NaiveBayesian.pkl',  # 0.00241
        'svm': './model/SVM_sklearn.pkl',
        'lrs': './model/Logistic_sklearn.pkl',
        'nbs': './model/Bayes_sklearn.pkl'
    }
    #
    cls_name = options.cls_name
    file_path = options.input_filename
    out_path = options.output_filename

    if cls_name not in classifiers.keys():
        print('check your classifiers name, you can use -h for help')
        sys.exit()

    start = time.time()
    jieba.initialize()
    try:
        with codecs.open(file_path, 'r', 'utf-8') as f:
            data = [line.strip() for line in f.read().split('\n')]
            if data[-1] == '':
                data.pop()
    except FileNotFoundError as e:
        print('Please check your input filename')
        sys.exit()

    # data = [Counter(d) for d in map(jieba.cut, data)]
    data = Pool().map(token, data)
    print('end token in {}\n'.format(time.time() - start))
    cv = BowTransform.load_vsm()
    data = cv.transform(data)
    print('end bow in {}\n'.format(time.time() - start))
    cls = joblib.load(classifiers[cls_name])
    predicted = cls.predict(data)

    # print(predicted)
    with open(out_path, 'w+') as f:
        for x in predicted:
            f.write(str(x) + '\n')
    print('task complete. total time: {}\n using {}'.format(time.time() - start, cls))


================================================
FILE: model/Bayes_sklearn.pkl
================================================
[File too large to display: 11.6 MB]

================================================
FILE: model/__init__.py
================================================
# -*- coding: utf-8 -*-
# @Date    : 2016/11/30
# @Author  : hrwhisper

================================================
FILE: model_manage.py
================================================
# -*- coding: utf-8 -*-
# @Date    : 2016/12/1
# @Author  : hrwhisper
from sklearn.externals import joblib


class BowTransform(object):
    default_path = './model/vsm.pkl'

    @staticmethod
    def save_vsm(model, filename=None):
        joblib.dump(model, filename if filename else BowTransform.default_path)

    @staticmethod
    def load_vsm(filename=None):
        return joblib.load(filename if filename else BowTransform.default_path)


class TrainData(object):
    default_path = './model/train_data.pkl'

    @staticmethod
    def save(model, filename=None):
        joblib.dump(model, filename if filename else TrainData.default_path)

    @staticmethod
    def load(filename=None):
        with open('./data/tags_token_results' + '_tag') as f:
            return joblib.load(filename if filename else TrainData.default_path), list(map(int, f.read().split('\n')[:-1]))


================================================
FILE: readme.md
================================================

## 环境 ##

- jieba
  - pip install jieba 
- sklearn
  - pip install -U scikit-learn
- python3.5



## 运行方法
- -c classfile_name 命令用来指定要用的分类器的名字：

| **名字**  | **对应的分类器**                  |
| ------- | --------------------------- |
| **p**   | Perceptron                  |
| **lr**  | LogisticRegression          |
| **nb**  | NaiveBayesian               |
| **svm** | SVM(sklearn)                |
| **lrs** | LogisticRegression(sklearn) |
| **nbs** | NaiveBayesian(sklearn)      |

- -i filename 为指定输入的短信文件名（该文件一行为一条短信）


- -o filename 为指定输出的结果文件（结果用0和1表示，1为垃圾短信，每一行对应输入文件的短信结果）

例如：

```
python judgeSpamMessage.py -c svm -i ./data/不带标签短信.txt -o ./data/result.txt
```

上述的命令指定了使用svm分类器，判断./data/不带标签短信.txt中的短信是否为垃圾短信，并将结果输出到./data/result.txt中。


## 训练啥的
- 首先运行token_and_save_to_file.py，分词保存结果
- test.py 中有交叉验证等方法



## 文件说明

文件夹解释如下：

| **文件夹名**       | **作用**      |
| -------------- | ----------- |
| **classifier** | 分类器代码存放的文件夹 |
| **data**       | 数据文件        |
| **model**      | 保存的模型       |

文件的解释如下：

| **文件夹名**                         | **作用**                                   |
| -------------------------------- | ---------------------------------------- |
| classifier/LogisticRegression.py | 本组实现的逻辑回归分类器源代码                          |
| classifier/NaiveBayesian.py      | 本组实现的朴素贝叶斯分类器源代码                         |
| classifier/Perceptron.py         | 本组实现的感知器分类器源代码                           |
| data/tags_token_results          | 带标签短信分词保存结果,token_and_save_to_file.py的生成的 |
| data/ tags_token_results_tag     | 带标签短信的类别                                 |
| data/不带标签短信.txt                  | 不带标签短信数据集                                |
| data/带标签短信.txt                   | 带标签短信数据集                                 |
| model/ Bayes_sklearn.pkl         | sklearn的贝叶斯分类器训练结果保存                     |
| model/ Logistic_sklearn.pkl      | sklearn的逻辑回归分类器训练结果保存                    |
| model/ LogisticRegression.pkl    | 本组实现的逻辑回归分类器训练结果保存                       |
| model/ NaiveBayesian.pkl         | 本组实现的贝叶斯分类器训练结果保存                        |
| model/ Perceptron.pkl            | 本组实现的感知器训练结果保存                           |
| model/ SVM_sklearn.pkl           | sklearn的SVM分类器结果保存                       |
| model/ train_data.pkl            | 带标签的短信的BOW表示结果                           |
| model/ vsm.pkl                   | 用于将新文档表示为BOW的训练完的类保存                     |
| judgeSpamMessage.py              | 用于判断输入的短信是否是垃圾短信                         |
| model_manage.py                  | 用于读入保存模型                                 |
| readme.md                        | 说明文件                                     |
| test.py                          | 测试文件                                     |
| token_and_save_to_file.py        | 分词并保存带标签的短信的结果，方便训练                      |

================================================
FILE: test.py
================================================
# -*- coding: utf-8 -*-
# @Date    : 2016/10/16
# @Author  : hrwhisper
import codecs
from collections import Counter
import datetime

from sklearn.externals import joblib
from sklearn.metrics import precision_recall_fscore_support
from sklearn.model_selection import cross_val_score
from sklearn.feature_extraction import DictVectorizer
import numpy as np
from sklearn import metrics, naive_bayes, svm, linear_model
from classifier.LogisticRegression import LogisticRegression
from classifier.NaiveBayesian import NaiveBayesian
from classifier.Perceptron import Perceptron
from model_manage import BowTransform, TrainData


def read_train_data():
    file_path = './data/tags_token_results'
    with codecs.open(file_path, 'r', 'utf-8') as f:
        data = [line.strip().split() for line in f.read().split('\n')]

    with open(file_path + '_tag') as f:
        return data[:-1], list(map(int, f.read().split('\n')[:-1]))


def _test(classifier, test_data, test_target):
    predicted = classifier.predict(test_data)
    print(predicted.shape)  # 160 1

    # print(sum(predicted == test_target), len(test_target), np.mean(predicted == test_target))
    print("Classification report for classifier %s:\n%s\n" % (
        classifier, metrics.classification_report(test_target, predicted, digits=4)))
    print("Confusion matrix:\n%s" % metrics.confusion_matrix(test_target, predicted))
    print(precision_recall_fscore_support(test_target, predicted))


def test_one(cls, use_save_data=True, train_cls=False, save_cls_path=None):
    if use_save_data:
        data, target = TrainData.load()
    else:
        data, target = read_train_data()
        # data_len = int(len(data) * 0.001)
        # data, target = data[:data_len], target[:data_len]

        data = [Counter(d) for d in data]  # 每一行为一个短信， 值就是TF
        # print(data[0])
        v = DictVectorizer()
        print('fit transform')
        data = v.fit_transform(data)  # 稀疏矩阵表示sparse matrix,词编好号
        TrainData.save(data)

    # print(data[0])
    data_len = data.shape[0]
    print('data', data.shape[1])
    end = int(0.8 * data_len)
    train_data, train_target = data[:end], target[:end]
    test_data, test_target = data[end:], target[end:]

    if train_cls:
        print('train classifier....')
        cls = cls.fit(train_data, train_target)
        print('train classifier complete')

    _test(cls, test_data, test_target)

    if save_cls_path:
        joblib.dump(cls, save_cls_path)


def cross_validation():
    data, target = TrainData.load()

    classifiers = {
        'Logistic by yhr': LogisticRegression(alpha=0.2, max_iter=2000),
        'Perceptron by yhr': Perceptron(alpha=0.1, max_iter=2000),
        'Bayesian by wqs': NaiveBayesian(),
        'Bernoulli Bayes from sklearn': naive_bayes.BernoulliNB(),
        'svm from sklearn': svm.LinearSVC(),
        'Logistic from sklearn': linear_model.LogisticRegression(),
        # 'decision tree':tree.DecisionTreeClassifier(),
    }

    for name, classifier in classifiers.items():
        this_scores = cross_val_score(classifier, data, target, cv=5, scoring='accuracy')
        print(name)
        print(this_scores)
        print(np.mean(this_scores))
        print(' ------------------------  \n\n')


def test_parameter():
    data, target = TrainData.load()
    max_score = 0
    max_alpha = max_iter = 0
    print('Perceptron')
    start = datetime.datetime.now()
    for alpha in [0.01, 0.1, 0.2]:  # 0.3 100 0.99133
        for iter in [100, 2000]:
            cls = Perceptron(alpha=alpha, max_iter=iter)
            this_scores = cross_val_score(cls, data, target, cv=5, scoring='accuracy')
            print(this_scores)
            cur = np.mean(this_scores)
            print(alpha, iter, cur)
            print(' ------------------------  \n\n')
            if cur > max_score:
                max_score = cur
                max_alpha, max_iter = alpha, iter
            print('current_max: ', max_score, max_alpha, max_iter)
    print((datetime.datetime.now() - start))


if __name__ == '__main__':
    # start = datetime.datetime.now()
    # test_one(LogisticRegression(alpha=0.1, max_iter=2000),
    #          train_cls=True, save_cls_path='./model/LogisticRegression.pkl')
    # print((datetime.datetime.now() - start))
    #
    # classifiers = {
    #     # 'Logistic by yhr': LogisticRegression(alpha=0.01, max_iter=200),
    #     # 'Perceptron by yhr': Perceptron(),
    #     # 'Bayesian by wqs': NaiveBayesian(),
    #     # 'Bayes_sklearn': naive_bayes.BernoulliNB(),
    #     # 'SVM_sklearn': svm.LinearSVC(),
    #     # 'Logistic_sklearn': linear_model.LogisticRegression(),
    #     # 'decision tree':tree.DecisionTreeClassifier(),
    # }
    # for name, cls in classifiers.items():
    #     test_one(cls, train_cls=True, save_cls_path='./model/' + name + '.pkl')


    #
    # test_one(LogisticRegression(alpha=0.2, max_iter=2000), train_cls=True,
    #          save_cls_path='./model/LogisticRegression.pkl')
    # test_one(Perceptron(alpha=0.1, max_iter=2000), train_cls=True, save_cls_path='./model/Perceptron.pkl')
    #
    classifiers = {
        'p': './model/Perceptron.pkl',  # 0.1 2000
        'lr': './model/LogisticRegression.pkl',  # 0.2 2000
        'nb': './model/NaiveBayesian.pkl',  # 0.00241
        'svm': './model/SVM_sklearn.pkl',
        'lrs': './model/Logistic_sklearn.pkl',
        'nbs': './model/Bayes_sklearn.pkl'
    }

    for _path in classifiers.values():
        cls = joblib.load(_path)
        test_one(cls)

        # cross_validation()
        # test_one(LogisticRegression(max_iter=100), train_cls=True)
        # test_parameter()


================================================
FILE: test_jieba.py
================================================
# -*- coding: utf-8 -*-
# @Date    : 2016/12/5
# @Author  : hrwhisper
import codecs
import datetime
from collections import Counter
import jieba
from multiprocessing import Pool


def test_not_tag_data():
    with codecs.open('./data/不带标签短信.txt', 'r', 'utf-8') as f:
        data = [line.strip() for line in f.read().split('\n')]
        if data[-1] == '':
            data.pop()

    return data


def f(x):
    return Counter(jieba.cut(x))


if __name__ == "__main__":
    # jieba.enable_parallel(2)
    start = datetime.datetime.now()
    data = test_not_tag_data()
    print('read data', datetime.datetime.now() - start)
    # data = [Counter(d) for d in map(jieba.cut, data)]
    res = Pool(4).map(f, data)
    # print(res)
    print('jieba ', datetime.datetime.now() - start)

    # cv = BowTransform.load_vsm()
    # data = cv.transform(data)
    # print('transform', datetime.datetime.now() - start)
    # cls = joblib.load(classifiers[cls_name])
    # print('load', datetime.datetime.now() - start)
    # predicted = cls.predict(data)
    #
    # print(predicted)
    # with open('./data/result.txt', 'w+') as f:
    #     for x in predicted:
    #         f.write(str(x) + '\n')

    # print(datetime.datetime.now() - start, ' %s' % cls)


================================================
FILE: test_judge.py
================================================
# -*- coding: utf-8 -*-
# @Date    : 2016/12/3
# @Author  : hrwhisper
import codecs
import sys
from collections import Counter
from multiprocessing import Pool
from optparse import OptionParser
import jieba
from sklearn import metrics
from sklearn.externals import joblib
from sklearn.feature_extraction import DictVectorizer
from sklearn.metrics import precision_recall_fscore_support
from model_manage import BowTransform
from test_jieba import test_not_tag_data


def test_data():
    def read_train_data():
        file_path = './data/带标签短信.txt'
        target = []
        data = []
        with codecs.open(file_path, 'r', 'utf-8') as f:
            for line in f.read().split('\n')[:-1]:
                line = line.strip()
                target.append(line[0])
                data.append(line[1:].lstrip())
        return data, target

    data, target = read_train_data()
    n = len(data) - int(0.8 * len(data))
    return data[-n:], list(map(int, target[-n:]))


def f(x):
    return Counter(jieba.cut(x))


if __name__ == "__main__":
    classifiers = {
        'p': './model/Perceptron.pkl',  # 0.1 2000
        'lr': './model/LogisticRegression.pkl',  # 0.2 2000
        'nb': './model/NaiveBayesian.pkl',  # 0.00241
        'svm': './model/SVM_sklearn.pkl',
        'lrs': './model/Logistic_sklearn.pkl',
        'nbs': './model/Bayes_sklearn.pkl'
    }
    import time

    # for cls_name in classifiers.keys():
    #     jieba.initialize()
    #
    #     start = time.time()
    #     data = test_not_tag_data()
    #     cls = joblib.load(classifiers[cls_name])
    #
    #     data = Pool().map(f, data)
    #     # data = [Counter(d) for d in map(jieba.cut, data)]
    #     print('end jieba', time.time() - start)
    #     cv = BowTransform.load_vsm()
    #     data = cv.transform(data)
    #     predicted = cls.predict(data)
    #     with open('./data/result.txt', 'w+') as f:
    #         for x in predicted:
    #             f.write(str(x) + '\n')
    #     print('end %s  with time:' % cls, time.time() - start)

    start = time.time()
    data, target = test_data()
    # data = [Counter(d) for d in map(jieba.cut, data)]
    data = Pool(4).map(f, data)
    cv = BowTransform.load_vsm()
    data = cv.transform(data)

    for cls_name in classifiers.keys():
        cls = joblib.load(classifiers[cls_name])
        predicted = cls.predict(data)

        print(predicted.shape)  # 160 1

        # print(sum(predicted == test_target), len(test_target), np.mean(predicted == test_target))
        print("Classification report for classifier %s:\n%s\n" % (
            cls, metrics.classification_report(target, predicted, digits=4)))
        print("Confusion matrix:\n%s" % metrics.confusion_matrix(target, predicted))
        print(precision_recall_fscore_support(target, predicted))
        print('end', time.time() - start)


================================================
FILE: test_judge2.py
================================================
# -*- coding: utf-8 -*-
# @Date    : 2016/12/3
# @Author  : hrwhisper
import codecs
import os
import sys
from collections import Counter
from multiprocessing import Pool
from optparse import OptionParser
import jieba
from sklearn import metrics
from sklearn.externals import joblib
from sklearn.metrics import precision_recall_fscore_support
from model_manage import BowTransform


def test_not_tag_data():
    with codecs.open('./data/不带标签短信.txt', 'r', 'utf-8') as f:
        data = f.read()
        return data
        #     if data[-1] == '':
        #         data.pop()
        # return data


def f(x):
    return Counter(jieba.cut(x))


if __name__ == "__main__":
    #
    classifiers = {
        'p': './model/Perceptron.pkl',  # 0.1 2000
        'lr': './model/LogisticRegression.pkl',  # 0.2 2000
        'nb': './model/NaiveBayesian.pkl',  # 0.00241
        'svm': './model/SVM_sklearn.pkl',
        'lrs': './model/Logistic_sklearn.pkl',
        'nbs': './model/Bayes_sklearn.pkl'
    }
    import time

    if os.name != 'nt':
        print('on linux enable parallel tantalization')
        jieba.enable_parallel(4)

    for cls_name in classifiers.keys():
        start = time.time()
        data = test_not_tag_data()
        cls = joblib.load(classifiers[cls_name])
        # data = Pool(4).map(f, data)

        data = [Counter(x) for x in ' '.join(jieba.cut(data)).split('\n')]
        # with codecs.open('./data/result.txt', 'w+', 'utf-8')  as f:
        #     f.write()

        print('end jieba', time.time() - start)
        cv = BowTransform.load_vsm()
        data = cv.transform(data)
        predicted = cls.predict(data)
        with open('./data/result.txt', 'w+') as f:
            for x in predicted:
                f.write(str(x) + '\n')
        print('end %s  with time:' % cls, time.time() - start)


        # data, target = test_data()
        # # data = [Counter(d) for d in map(jieba.cut, data)]
        # data = Pool(4).map(f, data)
        # cv = BowTransform.load_vsm()
        # data = cv.transform(data)


        # for cls_name in classifiers.keys():
        #     cls = joblib.load(classifiers[cls_name])
        #     predicted = cls.predict(data)
        #
        #     print(predicted.shape)  # 160 1
        #
        #     # print(sum(predicted == test_target), len(test_target), np.mean(predicted == test_target))
        #     print("Classification report for classifier %s:\n%s\n" % (
        #         cls, metrics.classification_report(target, predicted, digits=4)))
        #     print("Confusion matrix:\n%s" % metrics.confusion_matrix(target, predicted))
        #     print(precision_recall_fscore_support(target, predicted))
        #     print('end', time.time() - start)


================================================
FILE: token_and_save_to_file.py
================================================
# -*- coding: utf-8 -*-
# @Date    : 2016/10/7
# @Author  : hrwhisper
from collections import Counter
from multiprocessing.pool import Pool
import jieba
import codecs

from sklearn.feature_extraction import DictVectorizer

from model_manage import TrainData


def read_train_data():
    file_path = './data/带标签短信.txt'
    target = []
    data = []
    with codecs.open(file_path, 'r', 'utf-8') as f:
        for line in f.read().split('\n')[:-1]:
            line = line.strip()
            target.append(line[0])
            data.append(line[1:].lstrip())
    return data, target


def save_tokenlization_result(data, target, file_path='./data/tags_token_results'):
    with codecs.open(file_path, 'w', 'utf-8') as f:
        for x in data:
            f.write(' '.join(x) + '\n')

    with open(file_path + '_tag', 'w') as f:
        for x in target:
            f.write(x + '\n')


if __name__ == '__main__':
    # seg_list = jieba.cut("我来到北京清华大学", cut_all=False)
    # print("Default Mode: " + "/ ".join(seg_list))  # 精确模式

    data, target = read_train_data()
    data = Pool().map(jieba.lcut, data)
    # data = jieba.lcut(data)
    save_tokenlization_result(data, target)

    with codecs.open('./data/tags_token_results', 'r', 'utf-8') as f:
        data = [line.strip().split() for line in f.read().split('\n')]
        if not data[-1]: data.pop()
        t = [Counter(d) for d in data]  # 每一行为一个短信， 值就是TF
        v = DictVectorizer()
        t = v.fit_transform(t)  # 稀疏矩阵表示sparse matrix,词编好号
        TrainData.save(t)