Repository: Cheneng/DPCNN
Branch: master
Commit: 5342a4703d9f
Files: 9
Total size: 6.4 KB

Directory structure:
gitextract_2a987nd9/

├── README.md
├── checkpoint/
│   └── .placeholder
├── config.py
├── data/
│   ├── __init__.py
│   └── dataset.py
├── main.py
└── model/
    ├── BasicModule.py
    ├── DPCNN.py
    └── __init__.py

================================================
FILE CONTENTS
================================================

================================================
FILE: README.md
================================================
# Deep Pyramid Convolutional Neural Networks for Text Categorization

> This is a simple version of the paper *Deep Pyramid Convolutional Neural Networks for Text Categorization*.


!['model'](./pictures/figure1.png)


You should rewrite the Dataset class in the data/dataset.py  
and put your data in '/data/train' or any other directory.

run by

```
python main.py --lr=0.001 --epoch=20 --batch_size=64 --gpu=0 --seed=0 --label_num=2			
```

## Evaluation 
> 	I run the model in a dataset about AD identify.  
	And make a comparition between the TextCNN, LSTM and our DPCNN. 
	 
Loss of **TextCNN** and **LSTM**.  
<img src="./pictures/textcnn.png" width="350" height="250">  <img src="./pictures/lstm.png" width="350" height="250"> 


Loss of **DPCNN**.  
<img src="./pictures/dpcnn.png" width="350" height="250">



================================================
FILE: checkpoint/.placeholder
================================================


================================================
FILE: config.py
================================================
# —*- coding: utf-8 -*-


class Config(object):
    def __init__(self, word_embedding_dimension=100, word_num=20000,
                 epoch=2000, sentence_max_size=40,
                 learning_rate=0.01, batch_size=1,
                 drop_out=0.5,
                 dict_size=50000,
                 bidirectional=False,
                 doc_len=40):
        self.word_embedding_dimension = word_embedding_dimension
        self.word_num = word_num
        self.epoch = epoch
        self.sentence_max_size = sentence_max_size                   # 句子长度
        self.lr = learning_rate
        self.batch_size = batch_size
        self.dict_size = dict_size
        self.drop_out = drop_out
        self.bidirectional = bidirectional
        self.doc_len = doc_len



================================================
FILE: data/__init__.py
================================================
from .dataset import *

================================================
FILE: data/dataset.py
================================================
from torch.utils import data
import os


class TextDataset(data.Dataset):

    def __init__(self, path):
        self.file_name = os.listdir(path)

    def __getitem__(self, index):
        return self.train_set[index], self.labels[index]

    def __len__(self):
        return len(self.train_set)




================================================
FILE: main.py
================================================
# -*- coding: utf-8 -*-

import torch
import torch.autograd as autograd
import torch.nn as nn
import torch.optim as optim
import torch.utils.data as data
from config import Config
from model import DPCNN
from data import TextDataset
import argparse

torch.manual_seed(1)

parser = argparse.ArgumentParser()
parser.add_argument('--lr', type=float, default=0.1)
parser.add_argument('--batch_size', type=int, default=16)
parser.add_argument('--epoch', type=int, default=20)
parser.add_argument('--gpu', type=int, default=0)
parser.add_argument('--out_channel', type=int, default=2)
parser.add_argument('--label_num', type=int, default=2)
parser.add_argument('--seed', type=int, default=1)
args = parser.parse_args()


torch.manual_seed(args.seed)

if torch.cuda.is_available():
    torch.cuda.set_device(args.gpu)

# Create the configuration
config = Config(sentence_max_size=50,
                batch_size=args.batch_size,
                word_num=11000,
                label_num=args.label_num,
                learning_rate=args.lr,
                cuda=args.gpu,
                epoch=args.epoch,
                out_channel=args.out_channel)

training_set = TextDataset(path='data/train')

training_iter = data.DataLoader(dataset=training_set,
                                batch_size=config.batch_size,
                                num_workers=2)


model = DPCNN(config)
embeds = nn.Embedding(config.word_num, config.word_embedding_dimension)

if torch.cuda.is_available():
    model.cuda()
    embeds = embeds.cuda()

criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=config.lr)

count = 0
loss_sum = 0
# Train the model
for epoch in range(config.epoch):
    for data, label in training_iter:
        if config.cuda and torch.cuda.is_available():
            data = data.cuda()
            labels = label.cuda()

        input_data = embeds(autograd.Variable(data))
        out = model(input_data)
        loss = criterion(out, autograd.Variable(label.float()))

        loss_sum += loss.data[0]
        count += 1

        if count % 100 == 0:
            print("epoch", epoch, end='  ')
            print("The loss is: %.5f" % (loss_sum/100))

            loss_sum = 0
            count = 0

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
    # save the model in every epoch
    model.save('checkpoints/epoch{}.ckpt'.format(epoch))



================================================
FILE: model/BasicModule.py
================================================
# -*- coding: utf-8 -*-
import torch
import torch.nn as nn


class BasicModule(nn.Module):
    def __init__(self):
        super(BasicModule, self).__init__()
        self.model_name = str(type(self))

    def load(self, path):
        self.load_state_dict(torch.load(path))

    def save(self, path):
        torch.save(self.state_dict(), path)

================================================
FILE: model/DPCNN.py
================================================
import torch
import torch.nn as nn
import torch.nn.functional as F
from .BasicModule import BasicModule


class DPCNN(BasicModule):
    """
    DPCNN for sentences classification.
    """
    def __init__(self, config):
        super(DPCNN, self).__init__()
        self.config = config
        self.channel_size = 250
        self.conv_region_embedding = nn.Conv2d(1, self.channel_size, (3, self.config.word_embedding_dimension), stride=1)
        self.conv3 = nn.Conv2d(self.channel_size, self.channel_size, (3, 1), stride=1)
        self.pooling = nn.MaxPool2d(kernel_size=(3, 1), stride=2)
        self.padding_conv = nn.ZeroPad2d((0, 0, 1, 1))
        self.padding_pool = nn.ZeroPad2d((0, 0, 0, 1))
        self.act_fun = nn.ReLU()
        self.linear_out = nn.Linear(2*self.channel_size, 2)

    def forward(self, x):
        batch = x.shape[0]

        # Region embedding
        x = self.conv_region_embedding(x)        # [batch_size, channel_size, length, 1]

        x = self.padding_conv(x)                      # pad保证等长卷积，先通过激活函数再卷积
        x = self.act_fun(x)
        x = self.conv3(x)
        x = self.padding_conv(x)
        x = self.act_fun(x)
        x = self.conv3(x)

        while x.size()[-2] > 2:
            x = self._block(x)

        x = x.view(batch, 2*self.channel_size)
        x = self.linear_out(x)

        return x

    def _block(self, x):
        # Pooling
        x = self.padding_pool(x)
        px = self.pooling(x)

        # Convolution
        x = self.padding_conv(px)
        x = F.relu(x)
        x = self.conv3(x)

        x = self.padding_conv(x)
        x = F.relu(x)
        x = self.conv3(x)

        # Short Cut
        x = x + px

        return x

    def predict(self, x):
        self.eval()
        out = self.forward(x)
        predict_labels = torch.max(out, 1)[1]
        self.train(mode=True)
        return predict_labels



================================================
FILE: model/__init__.py
================================================