[ { "path": ".gitignore", "content": "*.pkl\n*.zip\ndata/\n.ipynb_checkpoints\n\n" }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2017 \n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "README.md", "content": "

\n\n--------------------------------------------------------------------------------\n\nThis repository provides tutorial code for deep learning researchers to learn [PyTorch](https://github.com/pytorch/pytorch). In the tutorial, most of the models were implemented with less than 30 lines of code. Before starting this tutorial, it is recommended to finish [Official Pytorch Tutorial](http://pytorch.org/tutorials/beginner/deep_learning_60min_blitz.html).\n\n\n
\n\n## Table of Contents\n\n#### 1. Basics\n* [PyTorch Basics](https://github.com/yunjey/pytorch-tutorial/tree/master/tutorials/01-basics/pytorch_basics/main.py)\n* [Linear Regression](https://github.com/yunjey/pytorch-tutorial/tree/master/tutorials/01-basics/linear_regression/main.py#L22-L23)\n* [Logistic Regression](https://github.com/yunjey/pytorch-tutorial/tree/master/tutorials/01-basics/logistic_regression/main.py#L33-L34)\n* [Feedforward Neural Network](https://github.com/yunjey/pytorch-tutorial/tree/master/tutorials/01-basics/feedforward_neural_network/main.py#L37-L49)\n\n#### 2. Intermediate\n* [Convolutional Neural Network](https://github.com/yunjey/pytorch-tutorial/tree/master/tutorials/02-intermediate/convolutional_neural_network/main.py#L35-L56)\n* [Deep Residual Network](https://github.com/yunjey/pytorch-tutorial/tree/master/tutorials/02-intermediate/deep_residual_network/main.py#L76-L113)\n* [Recurrent Neural Network](https://github.com/yunjey/pytorch-tutorial/tree/master/tutorials/02-intermediate/recurrent_neural_network/main.py#L39-L58)\n* [Bidirectional Recurrent Neural Network](https://github.com/yunjey/pytorch-tutorial/tree/master/tutorials/02-intermediate/bidirectional_recurrent_neural_network/main.py#L39-L58)\n* [Language Model (RNN-LM)](https://github.com/yunjey/pytorch-tutorial/tree/master/tutorials/02-intermediate/language_model/main.py#L30-L50)\n\n#### 3. Advanced\n* [Generative Adversarial Networks](https://github.com/yunjey/pytorch-tutorial/blob/master/tutorials/03-advanced/generative_adversarial_network/main.py#L41-L57)\n* [Variational Auto-Encoder](https://github.com/yunjey/pytorch-tutorial/blob/master/tutorials/03-advanced/variational_autoencoder/main.py#L38-L65)\n* [Neural Style Transfer](https://github.com/yunjey/pytorch-tutorial/tree/master/tutorials/03-advanced/neural_style_transfer)\n* [Image Captioning (CNN-RNN)](https://github.com/yunjey/pytorch-tutorial/tree/master/tutorials/03-advanced/image_captioning)\n\n#### 4. Utilities\n* [TensorBoard in PyTorch](https://github.com/yunjey/pytorch-tutorial/tree/master/tutorials/04-utils/tensorboard)\n\n\n
\n\n## Getting Started\n```bash\n$ git clone https://github.com/yunjey/pytorch-tutorial.git\n$ cd pytorch-tutorial/tutorials/PATH_TO_PROJECT\n$ python main.py\n```\n\n
\n\n## Dependencies\n* [Python 2.7 or 3.5+](https://www.continuum.io/downloads)\n* [PyTorch 0.4.0+](http://pytorch.org/)\n\n\n\n\n" }, { "path": "tutorials/01-basics/feedforward_neural_network/main.py", "content": "import torch\nimport torch.nn as nn\nimport torchvision\nimport torchvision.transforms as transforms\n\n\n# Device configuration\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Hyper-parameters \ninput_size = 784\nhidden_size = 500\nnum_classes = 10\nnum_epochs = 5\nbatch_size = 100\nlearning_rate = 0.001\n\n# MNIST dataset \ntrain_dataset = torchvision.datasets.MNIST(root='../../data', \n train=True, \n transform=transforms.ToTensor(), \n download=True)\n\ntest_dataset = torchvision.datasets.MNIST(root='../../data', \n train=False, \n transform=transforms.ToTensor())\n\n# Data loader\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n batch_size=batch_size, \n shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n batch_size=batch_size, \n shuffle=False)\n\n# Fully connected neural network with one hidden layer\nclass NeuralNet(nn.Module):\n def __init__(self, input_size, hidden_size, num_classes):\n super(NeuralNet, self).__init__()\n self.fc1 = nn.Linear(input_size, hidden_size) \n self.relu = nn.ReLU()\n self.fc2 = nn.Linear(hidden_size, num_classes) \n \n def forward(self, x):\n out = self.fc1(x)\n out = self.relu(out)\n out = self.fc2(out)\n return out\n\nmodel = NeuralNet(input_size, hidden_size, num_classes).to(device)\n\n# Loss and optimizer\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) \n\n# Train the model\ntotal_step = len(train_loader)\nfor epoch in range(num_epochs):\n for i, (images, labels) in enumerate(train_loader): \n # Move tensors to the configured device\n images = images.reshape(-1, 28*28).to(device)\n labels = labels.to(device)\n \n # Forward pass\n outputs = model(images)\n loss = criterion(outputs, labels)\n \n # Backward and optimize\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n \n if (i+1) % 100 == 0:\n print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}' \n .format(epoch+1, num_epochs, i+1, total_step, loss.item()))\n\n# Test the model\n# In test phase, we don't need to compute gradients (for memory efficiency)\nwith torch.no_grad():\n correct = 0\n total = 0\n for images, labels in test_loader:\n images = images.reshape(-1, 28*28).to(device)\n labels = labels.to(device)\n outputs = model(images)\n _, predicted = torch.max(outputs.data, 1)\n total += labels.size(0)\n correct += (predicted == labels).sum().item()\n\n print('Accuracy of the network on the 10000 test images: {} %'.format(100 * correct / total))\n\n# Save the model checkpoint\ntorch.save(model.state_dict(), 'model.ckpt')" }, { "path": "tutorials/01-basics/linear_regression/main.py", "content": "import torch\nimport torch.nn as nn\nimport numpy as np\nimport matplotlib.pyplot as plt\n\n\n# Hyper-parameters\ninput_size = 1\noutput_size = 1\nnum_epochs = 60\nlearning_rate = 0.001\n\n# Toy dataset\nx_train = np.array([[3.3], [4.4], [5.5], [6.71], [6.93], [4.168], \n [9.779], [6.182], [7.59], [2.167], [7.042], \n [10.791], [5.313], [7.997], [3.1]], dtype=np.float32)\n\ny_train = np.array([[1.7], [2.76], [2.09], [3.19], [1.694], [1.573], \n [3.366], [2.596], [2.53], [1.221], [2.827], \n [3.465], [1.65], [2.904], [1.3]], dtype=np.float32)\n\n# Linear regression model\nmodel = nn.Linear(input_size, output_size)\n\n# Loss and optimizer\ncriterion = nn.MSELoss()\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) \n\n# Train the model\nfor epoch in range(num_epochs):\n # Convert numpy arrays to torch tensors\n inputs = torch.from_numpy(x_train)\n targets = torch.from_numpy(y_train)\n\n # Forward pass\n outputs = model(inputs)\n loss = criterion(outputs, targets)\n \n # Backward and optimize\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n \n if (epoch+1) % 5 == 0:\n print ('Epoch [{}/{}], Loss: {:.4f}'.format(epoch+1, num_epochs, loss.item()))\n\n# Plot the graph\npredicted = model(torch.from_numpy(x_train)).detach().numpy()\nplt.plot(x_train, y_train, 'ro', label='Original data')\nplt.plot(x_train, predicted, label='Fitted line')\nplt.legend()\nplt.show()\n\n# Save the model checkpoint\ntorch.save(model.state_dict(), 'model.ckpt')" }, { "path": "tutorials/01-basics/logistic_regression/main.py", "content": "import torch\nimport torch.nn as nn\nimport torchvision\nimport torchvision.transforms as transforms\n\n\n# Hyper-parameters \ninput_size = 28 * 28 # 784\nnum_classes = 10\nnum_epochs = 5\nbatch_size = 100\nlearning_rate = 0.001\n\n# MNIST dataset (images and labels)\ntrain_dataset = torchvision.datasets.MNIST(root='../../data', \n train=True, \n transform=transforms.ToTensor(),\n download=True)\n\ntest_dataset = torchvision.datasets.MNIST(root='../../data', \n train=False, \n transform=transforms.ToTensor())\n\n# Data loader (input pipeline)\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n batch_size=batch_size, \n shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n batch_size=batch_size, \n shuffle=False)\n\n# Logistic regression model\nmodel = nn.Linear(input_size, num_classes)\n\n# Loss and optimizer\n# nn.CrossEntropyLoss() computes softmax internally\ncriterion = nn.CrossEntropyLoss() \noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) \n\n# Train the model\ntotal_step = len(train_loader)\nfor epoch in range(num_epochs):\n for i, (images, labels) in enumerate(train_loader):\n # Reshape images to (batch_size, input_size)\n images = images.reshape(-1, input_size)\n \n # Forward pass\n outputs = model(images)\n loss = criterion(outputs, labels)\n \n # Backward and optimize\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n \n if (i+1) % 100 == 0:\n print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}' \n .format(epoch+1, num_epochs, i+1, total_step, loss.item()))\n\n# Test the model\n# In test phase, we don't need to compute gradients (for memory efficiency)\nwith torch.no_grad():\n correct = 0\n total = 0\n for images, labels in test_loader:\n images = images.reshape(-1, input_size)\n outputs = model(images)\n _, predicted = torch.max(outputs.data, 1)\n total += labels.size(0)\n correct += (predicted == labels).sum()\n\n print('Accuracy of the model on the 10000 test images: {} %'.format(100 * correct / total))\n\n# Save the model checkpoint\ntorch.save(model.state_dict(), 'model.ckpt')\n" }, { "path": "tutorials/01-basics/pytorch_basics/main.py", "content": "import torch \nimport torchvision\nimport torch.nn as nn\nimport numpy as np\nimport torchvision.transforms as transforms\n\n\n# ================================================================== #\n# Table of Contents #\n# ================================================================== #\n\n# 1. Basic autograd example 1 (Line 25 to 39)\n# 2. Basic autograd example 2 (Line 46 to 83)\n# 3. Loading data from numpy (Line 90 to 97)\n# 4. Input pipline (Line 104 to 129)\n# 5. Input pipline for custom dataset (Line 136 to 156)\n# 6. Pretrained model (Line 163 to 176)\n# 7. Save and load model (Line 183 to 189) \n\n\n# ================================================================== #\n# 1. Basic autograd example 1 #\n# ================================================================== #\n\n# Create tensors.\nx = torch.tensor(1., requires_grad=True)\nw = torch.tensor(2., requires_grad=True)\nb = torch.tensor(3., requires_grad=True)\n\n# Build a computational graph.\ny = w * x + b # y = 2 * x + 3\n\n# Compute gradients.\ny.backward()\n\n# Print out the gradients.\nprint(x.grad) # x.grad = 2 \nprint(w.grad) # w.grad = 1 \nprint(b.grad) # b.grad = 1 \n\n\n# ================================================================== #\n# 2. Basic autograd example 2 #\n# ================================================================== #\n\n# Create tensors of shape (10, 3) and (10, 2).\nx = torch.randn(10, 3)\ny = torch.randn(10, 2)\n\n# Build a fully connected layer.\nlinear = nn.Linear(3, 2)\nprint ('w: ', linear.weight)\nprint ('b: ', linear.bias)\n\n# Build loss function and optimizer.\ncriterion = nn.MSELoss()\noptimizer = torch.optim.SGD(linear.parameters(), lr=0.01)\n\n# Forward pass.\npred = linear(x)\n\n# Compute loss.\nloss = criterion(pred, y)\nprint('loss: ', loss.item())\n\n# Backward pass.\nloss.backward()\n\n# Print out the gradients.\nprint ('dL/dw: ', linear.weight.grad) \nprint ('dL/db: ', linear.bias.grad)\n\n# 1-step gradient descent.\noptimizer.step()\n\n# You can also perform gradient descent at the low level.\n# linear.weight.data.sub_(0.01 * linear.weight.grad.data)\n# linear.bias.data.sub_(0.01 * linear.bias.grad.data)\n\n# Print out the loss after 1-step gradient descent.\npred = linear(x)\nloss = criterion(pred, y)\nprint('loss after 1 step optimization: ', loss.item())\n\n\n# ================================================================== #\n# 3. Loading data from numpy #\n# ================================================================== #\n\n# Create a numpy array.\nx = np.array([[1, 2], [3, 4]])\n\n# Convert the numpy array to a torch tensor.\ny = torch.from_numpy(x)\n\n# Convert the torch tensor to a numpy array.\nz = y.numpy()\n\n\n# ================================================================== #\n# 4. Input pipeline #\n# ================================================================== #\n\n# Download and construct CIFAR-10 dataset.\ntrain_dataset = torchvision.datasets.CIFAR10(root='../../data/',\n train=True, \n transform=transforms.ToTensor(),\n download=True)\n\n# Fetch one data pair (read data from disk).\nimage, label = train_dataset[0]\nprint (image.size())\nprint (label)\n\n# Data loader (this provides queues and threads in a very simple way).\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n batch_size=64, \n shuffle=True)\n\n# When iteration starts, queue and thread start to load data from files.\ndata_iter = iter(train_loader)\n\n# Mini-batch images and labels.\nimages, labels = data_iter.next()\n\n# Actual usage of the data loader is as below.\nfor images, labels in train_loader:\n # Training code should be written here.\n pass\n\n\n# ================================================================== #\n# 5. Input pipeline for custom dataset #\n# ================================================================== #\n\n# You should build your custom dataset as below.\nclass CustomDataset(torch.utils.data.Dataset):\n def __init__(self):\n # TODO\n # 1. Initialize file paths or a list of file names. \n pass\n def __getitem__(self, index):\n # TODO\n # 1. Read one data from file (e.g. using numpy.fromfile, PIL.Image.open).\n # 2. Preprocess the data (e.g. torchvision.Transform).\n # 3. Return a data pair (e.g. image and label).\n pass\n def __len__(self):\n # You should change 0 to the total size of your dataset.\n return 0 \n\n# You can then use the prebuilt data loader. \ncustom_dataset = CustomDataset()\ntrain_loader = torch.utils.data.DataLoader(dataset=custom_dataset,\n batch_size=64, \n shuffle=True)\n\n\n# ================================================================== #\n# 6. Pretrained model #\n# ================================================================== #\n\n# Download and load the pretrained ResNet-18.\nresnet = torchvision.models.resnet18(pretrained=True)\n\n# If you want to finetune only the top layer of the model, set as below.\nfor param in resnet.parameters():\n param.requires_grad = False\n\n# Replace the top layer for finetuning.\nresnet.fc = nn.Linear(resnet.fc.in_features, 100) # 100 is an example.\n\n# Forward pass.\nimages = torch.randn(64, 3, 224, 224)\noutputs = resnet(images)\nprint (outputs.size()) # (64, 100)\n\n\n# ================================================================== #\n# 7. Save and load the model #\n# ================================================================== #\n\n# Save and load the entire model.\ntorch.save(resnet, 'model.ckpt')\nmodel = torch.load('model.ckpt')\n\n# Save and load only the model parameters (recommended).\ntorch.save(resnet.state_dict(), 'params.ckpt')\nresnet.load_state_dict(torch.load('params.ckpt'))\n" }, { "path": "tutorials/02-intermediate/bidirectional_recurrent_neural_network/main.py", "content": "import torch \nimport torch.nn as nn\nimport torchvision\nimport torchvision.transforms as transforms\n\n\n# Device configuration\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Hyper-parameters\nsequence_length = 28\ninput_size = 28\nhidden_size = 128\nnum_layers = 2\nnum_classes = 10\nbatch_size = 100\nnum_epochs = 2\nlearning_rate = 0.003\n\n# MNIST dataset\ntrain_dataset = torchvision.datasets.MNIST(root='../../data/',\n train=True, \n transform=transforms.ToTensor(),\n download=True)\n\ntest_dataset = torchvision.datasets.MNIST(root='../../data/',\n train=False, \n transform=transforms.ToTensor())\n\n# Data loader\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n batch_size=batch_size, \n shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n batch_size=batch_size, \n shuffle=False)\n\n# Bidirectional recurrent neural network (many-to-one)\nclass BiRNN(nn.Module):\n def __init__(self, input_size, hidden_size, num_layers, num_classes):\n super(BiRNN, self).__init__()\n self.hidden_size = hidden_size\n self.num_layers = num_layers\n self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True, bidirectional=True)\n self.fc = nn.Linear(hidden_size*2, num_classes) # 2 for bidirection\n \n def forward(self, x):\n # Set initial states\n h0 = torch.zeros(self.num_layers*2, x.size(0), self.hidden_size).to(device) # 2 for bidirection \n c0 = torch.zeros(self.num_layers*2, x.size(0), self.hidden_size).to(device)\n \n # Forward propagate LSTM\n out, _ = self.lstm(x, (h0, c0)) # out: tensor of shape (batch_size, seq_length, hidden_size*2)\n \n # Decode the hidden state of the last time step\n out = self.fc(out[:, -1, :])\n return out\n\nmodel = BiRNN(input_size, hidden_size, num_layers, num_classes).to(device)\n\n\n# Loss and optimizer\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n \n# Train the model\ntotal_step = len(train_loader)\nfor epoch in range(num_epochs):\n for i, (images, labels) in enumerate(train_loader):\n images = images.reshape(-1, sequence_length, input_size).to(device)\n labels = labels.to(device)\n \n # Forward pass\n outputs = model(images)\n loss = criterion(outputs, labels)\n \n # Backward and optimize\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n \n if (i+1) % 100 == 0:\n print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}' \n .format(epoch+1, num_epochs, i+1, total_step, loss.item()))\n\n# Test the model\nwith torch.no_grad():\n correct = 0\n total = 0\n for images, labels in test_loader:\n images = images.reshape(-1, sequence_length, input_size).to(device)\n labels = labels.to(device)\n outputs = model(images)\n _, predicted = torch.max(outputs.data, 1)\n total += labels.size(0)\n correct += (predicted == labels).sum().item()\n\n print('Test Accuracy of the model on the 10000 test images: {} %'.format(100 * correct / total)) \n\n# Save the model checkpoint\ntorch.save(model.state_dict(), 'model.ckpt')" }, { "path": "tutorials/02-intermediate/convolutional_neural_network/main.py", "content": "import torch \nimport torch.nn as nn\nimport torchvision\nimport torchvision.transforms as transforms\n\n\n# Device configuration\ndevice = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n\n# Hyper parameters\nnum_epochs = 5\nnum_classes = 10\nbatch_size = 100\nlearning_rate = 0.001\n\n# MNIST dataset\ntrain_dataset = torchvision.datasets.MNIST(root='../../data/',\n train=True, \n transform=transforms.ToTensor(),\n download=True)\n\ntest_dataset = torchvision.datasets.MNIST(root='../../data/',\n train=False, \n transform=transforms.ToTensor())\n\n# Data loader\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n batch_size=batch_size, \n shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n batch_size=batch_size, \n shuffle=False)\n\n# Convolutional neural network (two convolutional layers)\nclass ConvNet(nn.Module):\n def __init__(self, num_classes=10):\n super(ConvNet, self).__init__()\n self.layer1 = nn.Sequential(\n nn.Conv2d(1, 16, kernel_size=5, stride=1, padding=2),\n nn.BatchNorm2d(16),\n nn.ReLU(),\n nn.MaxPool2d(kernel_size=2, stride=2))\n self.layer2 = nn.Sequential(\n nn.Conv2d(16, 32, kernel_size=5, stride=1, padding=2),\n nn.BatchNorm2d(32),\n nn.ReLU(),\n nn.MaxPool2d(kernel_size=2, stride=2))\n self.fc = nn.Linear(7*7*32, num_classes)\n \n def forward(self, x):\n out = self.layer1(x)\n out = self.layer2(out)\n out = out.reshape(out.size(0), -1)\n out = self.fc(out)\n return out\n\nmodel = ConvNet(num_classes).to(device)\n\n# Loss and optimizer\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n\n# Train the model\ntotal_step = len(train_loader)\nfor epoch in range(num_epochs):\n for i, (images, labels) in enumerate(train_loader):\n images = images.to(device)\n labels = labels.to(device)\n \n # Forward pass\n outputs = model(images)\n loss = criterion(outputs, labels)\n \n # Backward and optimize\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n \n if (i+1) % 100 == 0:\n print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}' \n .format(epoch+1, num_epochs, i+1, total_step, loss.item()))\n\n# Test the model\nmodel.eval() # eval mode (batchnorm uses moving mean/variance instead of mini-batch mean/variance)\nwith torch.no_grad():\n correct = 0\n total = 0\n for images, labels in test_loader:\n images = images.to(device)\n labels = labels.to(device)\n outputs = model(images)\n _, predicted = torch.max(outputs.data, 1)\n total += labels.size(0)\n correct += (predicted == labels).sum().item()\n\n print('Test Accuracy of the model on the 10000 test images: {} %'.format(100 * correct / total))\n\n# Save the model checkpoint\ntorch.save(model.state_dict(), 'model.ckpt')" }, { "path": "tutorials/02-intermediate/deep_residual_network/main.py", "content": "# ---------------------------------------------------------------------------- #\n# An implementation of https://arxiv.org/pdf/1512.03385.pdf #\n# See section 4.2 for the model architecture on CIFAR-10 #\n# Some part of the code was referenced from below #\n# https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py #\n# ---------------------------------------------------------------------------- #\n\nimport torch\nimport torch.nn as nn\nimport torchvision\nimport torchvision.transforms as transforms\n\n\n# Device configuration\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Hyper-parameters\nnum_epochs = 80\nbatch_size = 100\nlearning_rate = 0.001\n\n# Image preprocessing modules\ntransform = transforms.Compose([\n transforms.Pad(4),\n transforms.RandomHorizontalFlip(),\n transforms.RandomCrop(32),\n transforms.ToTensor()])\n\n# CIFAR-10 dataset\ntrain_dataset = torchvision.datasets.CIFAR10(root='../../data/',\n train=True, \n transform=transform,\n download=True)\n\ntest_dataset = torchvision.datasets.CIFAR10(root='../../data/',\n train=False, \n transform=transforms.ToTensor())\n\n# Data loader\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n batch_size=batch_size,\n shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n batch_size=batch_size,\n shuffle=False)\n\n# 3x3 convolution\ndef conv3x3(in_channels, out_channels, stride=1):\n return nn.Conv2d(in_channels, out_channels, kernel_size=3, \n stride=stride, padding=1, bias=False)\n\n# Residual block\nclass ResidualBlock(nn.Module):\n def __init__(self, in_channels, out_channels, stride=1, downsample=None):\n super(ResidualBlock, self).__init__()\n self.conv1 = conv3x3(in_channels, out_channels, stride)\n self.bn1 = nn.BatchNorm2d(out_channels)\n self.relu = nn.ReLU(inplace=True)\n self.conv2 = conv3x3(out_channels, out_channels)\n self.bn2 = nn.BatchNorm2d(out_channels)\n self.downsample = downsample\n \n def forward(self, x):\n residual = x\n out = self.conv1(x)\n out = self.bn1(out)\n out = self.relu(out)\n out = self.conv2(out)\n out = self.bn2(out)\n if self.downsample:\n residual = self.downsample(x)\n out += residual\n out = self.relu(out)\n return out\n\n# ResNet\nclass ResNet(nn.Module):\n def __init__(self, block, layers, num_classes=10):\n super(ResNet, self).__init__()\n self.in_channels = 16\n self.conv = conv3x3(3, 16)\n self.bn = nn.BatchNorm2d(16)\n self.relu = nn.ReLU(inplace=True)\n self.layer1 = self.make_layer(block, 16, layers[0])\n self.layer2 = self.make_layer(block, 32, layers[1], 2)\n self.layer3 = self.make_layer(block, 64, layers[2], 2)\n self.avg_pool = nn.AvgPool2d(8)\n self.fc = nn.Linear(64, num_classes)\n \n def make_layer(self, block, out_channels, blocks, stride=1):\n downsample = None\n if (stride != 1) or (self.in_channels != out_channels):\n downsample = nn.Sequential(\n conv3x3(self.in_channels, out_channels, stride=stride),\n nn.BatchNorm2d(out_channels))\n layers = []\n layers.append(block(self.in_channels, out_channels, stride, downsample))\n self.in_channels = out_channels\n for i in range(1, blocks):\n layers.append(block(out_channels, out_channels))\n return nn.Sequential(*layers)\n \n def forward(self, x):\n out = self.conv(x)\n out = self.bn(out)\n out = self.relu(out)\n out = self.layer1(out)\n out = self.layer2(out)\n out = self.layer3(out)\n out = self.avg_pool(out)\n out = out.view(out.size(0), -1)\n out = self.fc(out)\n return out\n \nmodel = ResNet(ResidualBlock, [2, 2, 2]).to(device)\n\n\n# Loss and optimizer\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n\n# For updating learning rate\ndef update_lr(optimizer, lr): \n for param_group in optimizer.param_groups:\n param_group['lr'] = lr\n\n# Train the model\ntotal_step = len(train_loader)\ncurr_lr = learning_rate\nfor epoch in range(num_epochs):\n for i, (images, labels) in enumerate(train_loader):\n images = images.to(device)\n labels = labels.to(device)\n \n # Forward pass\n outputs = model(images)\n loss = criterion(outputs, labels)\n \n # Backward and optimize\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n \n if (i+1) % 100 == 0:\n print (\"Epoch [{}/{}], Step [{}/{}] Loss: {:.4f}\"\n .format(epoch+1, num_epochs, i+1, total_step, loss.item()))\n\n # Decay learning rate\n if (epoch+1) % 20 == 0:\n curr_lr /= 3\n update_lr(optimizer, curr_lr)\n\n# Test the model\nmodel.eval()\nwith torch.no_grad():\n correct = 0\n total = 0\n for images, labels in test_loader:\n images = images.to(device)\n labels = labels.to(device)\n outputs = model(images)\n _, predicted = torch.max(outputs.data, 1)\n total += labels.size(0)\n correct += (predicted == labels).sum().item()\n\n print('Accuracy of the model on the test images: {} %'.format(100 * correct / total))\n\n# Save the model checkpoint\ntorch.save(model.state_dict(), 'resnet.ckpt')\n" }, { "path": "tutorials/02-intermediate/language_model/data_utils.py", "content": "import torch\nimport os\n\n\nclass Dictionary(object):\n def __init__(self):\n self.word2idx = {}\n self.idx2word = {}\n self.idx = 0\n \n def add_word(self, word):\n if not word in self.word2idx:\n self.word2idx[word] = self.idx\n self.idx2word[self.idx] = word\n self.idx += 1\n \n def __len__(self):\n return len(self.word2idx)\n\n\nclass Corpus(object):\n def __init__(self):\n self.dictionary = Dictionary()\n\n def get_data(self, path, batch_size=20):\n # Add words to the dictionary\n with open(path, 'r') as f:\n tokens = 0\n for line in f:\n words = line.split() + ['']\n tokens += len(words)\n for word in words: \n self.dictionary.add_word(word) \n \n # Tokenize the file content\n ids = torch.LongTensor(tokens)\n token = 0\n with open(path, 'r') as f:\n for line in f:\n words = line.split() + ['']\n for word in words:\n ids[token] = self.dictionary.word2idx[word]\n token += 1\n num_batches = ids.size(0) // batch_size\n ids = ids[:num_batches*batch_size]\n return ids.view(batch_size, -1)" }, { "path": "tutorials/02-intermediate/language_model/main.py", "content": "# Some part of the code was referenced from below.\n# https://github.com/pytorch/examples/tree/master/word_language_model \nimport torch\nimport torch.nn as nn\nimport numpy as np\nfrom torch.nn.utils import clip_grad_norm_\nfrom data_utils import Dictionary, Corpus\n\n\n# Device configuration\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Hyper-parameters\nembed_size = 128\nhidden_size = 1024\nnum_layers = 1\nnum_epochs = 5\nnum_samples = 1000 # number of words to be sampled\nbatch_size = 20\nseq_length = 30\nlearning_rate = 0.002\n\n# Load \"Penn Treebank\" dataset\ncorpus = Corpus()\nids = corpus.get_data('data/train.txt', batch_size)\nvocab_size = len(corpus.dictionary)\nnum_batches = ids.size(1) // seq_length\n\n\n# RNN based language model\nclass RNNLM(nn.Module):\n def __init__(self, vocab_size, embed_size, hidden_size, num_layers):\n super(RNNLM, self).__init__()\n self.embed = nn.Embedding(vocab_size, embed_size)\n self.lstm = nn.LSTM(embed_size, hidden_size, num_layers, batch_first=True)\n self.linear = nn.Linear(hidden_size, vocab_size)\n \n def forward(self, x, h):\n # Embed word ids to vectors\n x = self.embed(x)\n \n # Forward propagate LSTM\n out, (h, c) = self.lstm(x, h)\n \n # Reshape output to (batch_size*sequence_length, hidden_size)\n out = out.reshape(out.size(0)*out.size(1), out.size(2))\n \n # Decode hidden states of all time steps\n out = self.linear(out)\n return out, (h, c)\n\nmodel = RNNLM(vocab_size, embed_size, hidden_size, num_layers).to(device)\n\n# Loss and optimizer\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n\n# Truncated backpropagation\ndef detach(states):\n return [state.detach() for state in states] \n\n# Train the model\nfor epoch in range(num_epochs):\n # Set initial hidden and cell states\n states = (torch.zeros(num_layers, batch_size, hidden_size).to(device),\n torch.zeros(num_layers, batch_size, hidden_size).to(device))\n \n for i in range(0, ids.size(1) - seq_length, seq_length):\n # Get mini-batch inputs and targets\n inputs = ids[:, i:i+seq_length].to(device)\n targets = ids[:, (i+1):(i+1)+seq_length].to(device)\n \n # Forward pass\n states = detach(states)\n outputs, states = model(inputs, states)\n loss = criterion(outputs, targets.reshape(-1))\n \n # Backward and optimize\n optimizer.zero_grad()\n loss.backward()\n clip_grad_norm_(model.parameters(), 0.5)\n optimizer.step()\n\n step = (i+1) // seq_length\n if step % 100 == 0:\n print ('Epoch [{}/{}], Step[{}/{}], Loss: {:.4f}, Perplexity: {:5.2f}'\n .format(epoch+1, num_epochs, step, num_batches, loss.item(), np.exp(loss.item())))\n\n# Test the model\nwith torch.no_grad():\n with open('sample.txt', 'w') as f:\n # Set intial hidden ane cell states\n state = (torch.zeros(num_layers, 1, hidden_size).to(device),\n torch.zeros(num_layers, 1, hidden_size).to(device))\n\n # Select one word id randomly\n prob = torch.ones(vocab_size)\n input = torch.multinomial(prob, num_samples=1).unsqueeze(1).to(device)\n\n for i in range(num_samples):\n # Forward propagate RNN \n output, state = model(input, state)\n\n # Sample a word id\n prob = output.exp()\n word_id = torch.multinomial(prob, num_samples=1).item()\n\n # Fill input with sampled word id for the next time step\n input.fill_(word_id)\n\n # File write\n word = corpus.dictionary.idx2word[word_id]\n word = '\\n' if word == '' else word + ' '\n f.write(word)\n\n if (i+1) % 100 == 0:\n print('Sampled [{}/{}] words and save to {}'.format(i+1, num_samples, 'sample.txt'))\n\n# Save the model checkpoints\ntorch.save(model.state_dict(), 'model.ckpt')" }, { "path": "tutorials/02-intermediate/recurrent_neural_network/main.py", "content": "import torch \nimport torch.nn as nn\nimport torchvision\nimport torchvision.transforms as transforms\n\n\n# Device configuration\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Hyper-parameters\nsequence_length = 28\ninput_size = 28\nhidden_size = 128\nnum_layers = 2\nnum_classes = 10\nbatch_size = 100\nnum_epochs = 2\nlearning_rate = 0.01\n\n# MNIST dataset\ntrain_dataset = torchvision.datasets.MNIST(root='../../data/',\n train=True, \n transform=transforms.ToTensor(),\n download=True)\n\ntest_dataset = torchvision.datasets.MNIST(root='../../data/',\n train=False, \n transform=transforms.ToTensor())\n\n# Data loader\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n batch_size=batch_size, \n shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n batch_size=batch_size, \n shuffle=False)\n\n# Recurrent neural network (many-to-one)\nclass RNN(nn.Module):\n def __init__(self, input_size, hidden_size, num_layers, num_classes):\n super(RNN, self).__init__()\n self.hidden_size = hidden_size\n self.num_layers = num_layers\n self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)\n self.fc = nn.Linear(hidden_size, num_classes)\n \n def forward(self, x):\n # Set initial hidden and cell states \n h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device) \n c0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device)\n \n # Forward propagate LSTM\n out, _ = self.lstm(x, (h0, c0)) # out: tensor of shape (batch_size, seq_length, hidden_size)\n \n # Decode the hidden state of the last time step\n out = self.fc(out[:, -1, :])\n return out\n\nmodel = RNN(input_size, hidden_size, num_layers, num_classes).to(device)\n\n\n# Loss and optimizer\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n\n# Train the model\ntotal_step = len(train_loader)\nfor epoch in range(num_epochs):\n for i, (images, labels) in enumerate(train_loader):\n images = images.reshape(-1, sequence_length, input_size).to(device)\n labels = labels.to(device)\n \n # Forward pass\n outputs = model(images)\n loss = criterion(outputs, labels)\n \n # Backward and optimize\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n \n if (i+1) % 100 == 0:\n print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}' \n .format(epoch+1, num_epochs, i+1, total_step, loss.item()))\n\n# Test the model\nmodel.eval()\nwith torch.no_grad():\n correct = 0\n total = 0\n for images, labels in test_loader:\n images = images.reshape(-1, sequence_length, input_size).to(device)\n labels = labels.to(device)\n outputs = model(images)\n _, predicted = torch.max(outputs.data, 1)\n total += labels.size(0)\n correct += (predicted == labels).sum().item()\n\n print('Test Accuracy of the model on the 10000 test images: {} %'.format(100 * correct / total)) \n\n# Save the model checkpoint\ntorch.save(model.state_dict(), 'model.ckpt')" }, { "path": "tutorials/03-advanced/generative_adversarial_network/main.py", "content": "import os\nimport torch\nimport torchvision\nimport torch.nn as nn\nfrom torchvision import transforms\nfrom torchvision.utils import save_image\n\n\n# Device configuration\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Hyper-parameters\nlatent_size = 64\nhidden_size = 256\nimage_size = 784\nnum_epochs = 200\nbatch_size = 100\nsample_dir = 'samples'\n\n# Create a directory if not exists\nif not os.path.exists(sample_dir):\n os.makedirs(sample_dir)\n\n# Image processing\n# transform = transforms.Compose([\n# transforms.ToTensor(),\n# transforms.Normalize(mean=(0.5, 0.5, 0.5), # 3 for RGB channels\n# std=(0.5, 0.5, 0.5))])\ntransform = transforms.Compose([\n transforms.ToTensor(),\n transforms.Normalize(mean=[0.5], # 1 for greyscale channels\n std=[0.5])])\n\n# MNIST dataset\nmnist = torchvision.datasets.MNIST(root='../../data/',\n train=True,\n transform=transform,\n download=True)\n\n# Data loader\ndata_loader = torch.utils.data.DataLoader(dataset=mnist,\n batch_size=batch_size, \n shuffle=True)\n\n# Discriminator\nD = nn.Sequential(\n nn.Linear(image_size, hidden_size),\n nn.LeakyReLU(0.2),\n nn.Linear(hidden_size, hidden_size),\n nn.LeakyReLU(0.2),\n nn.Linear(hidden_size, 1),\n nn.Sigmoid())\n\n# Generator \nG = nn.Sequential(\n nn.Linear(latent_size, hidden_size),\n nn.ReLU(),\n nn.Linear(hidden_size, hidden_size),\n nn.ReLU(),\n nn.Linear(hidden_size, image_size),\n nn.Tanh())\n\n# Device setting\nD = D.to(device)\nG = G.to(device)\n\n# Binary cross entropy loss and optimizer\ncriterion = nn.BCELoss()\nd_optimizer = torch.optim.Adam(D.parameters(), lr=0.0002)\ng_optimizer = torch.optim.Adam(G.parameters(), lr=0.0002)\n\ndef denorm(x):\n out = (x + 1) / 2\n return out.clamp(0, 1)\n\ndef reset_grad():\n d_optimizer.zero_grad()\n g_optimizer.zero_grad()\n\n# Start training\ntotal_step = len(data_loader)\nfor epoch in range(num_epochs):\n for i, (images, _) in enumerate(data_loader):\n images = images.reshape(batch_size, -1).to(device)\n \n # Create the labels which are later used as input for the BCE loss\n real_labels = torch.ones(batch_size, 1).to(device)\n fake_labels = torch.zeros(batch_size, 1).to(device)\n\n # ================================================================== #\n # Train the discriminator #\n # ================================================================== #\n\n # Compute BCE_Loss using real images where BCE_Loss(x, y): - y * log(D(x)) - (1-y) * log(1 - D(x))\n # Second term of the loss is always zero since real_labels == 1\n outputs = D(images)\n d_loss_real = criterion(outputs, real_labels)\n real_score = outputs\n \n # Compute BCELoss using fake images\n # First term of the loss is always zero since fake_labels == 0\n z = torch.randn(batch_size, latent_size).to(device)\n fake_images = G(z)\n outputs = D(fake_images)\n d_loss_fake = criterion(outputs, fake_labels)\n fake_score = outputs\n \n # Backprop and optimize\n d_loss = d_loss_real + d_loss_fake\n reset_grad()\n d_loss.backward()\n d_optimizer.step()\n \n # ================================================================== #\n # Train the generator #\n # ================================================================== #\n\n # Compute loss with fake images\n z = torch.randn(batch_size, latent_size).to(device)\n fake_images = G(z)\n outputs = D(fake_images)\n \n # We train G to maximize log(D(G(z)) instead of minimizing log(1-D(G(z)))\n # For the reason, see the last paragraph of section 3. https://arxiv.org/pdf/1406.2661.pdf\n g_loss = criterion(outputs, real_labels)\n \n # Backprop and optimize\n reset_grad()\n g_loss.backward()\n g_optimizer.step()\n \n if (i+1) % 200 == 0:\n print('Epoch [{}/{}], Step [{}/{}], d_loss: {:.4f}, g_loss: {:.4f}, D(x): {:.2f}, D(G(z)): {:.2f}' \n .format(epoch, num_epochs, i+1, total_step, d_loss.item(), g_loss.item(), \n real_score.mean().item(), fake_score.mean().item()))\n \n # Save real images\n if (epoch+1) == 1:\n images = images.reshape(images.size(0), 1, 28, 28)\n save_image(denorm(images), os.path.join(sample_dir, 'real_images.png'))\n \n # Save sampled images\n fake_images = fake_images.reshape(fake_images.size(0), 1, 28, 28)\n save_image(denorm(fake_images), os.path.join(sample_dir, 'fake_images-{}.png'.format(epoch+1)))\n\n# Save the model checkpoints \ntorch.save(G.state_dict(), 'G.ckpt')\ntorch.save(D.state_dict(), 'D.ckpt')" }, { "path": "tutorials/03-advanced/image_captioning/README.md", "content": "# Image Captioning\nThe goal of image captioning is to convert a given input image into a natural language description. The encoder-decoder framework is widely used for this task. The image encoder is a convolutional neural network (CNN). In this tutorial, we used [resnet-152](https://arxiv.org/abs/1512.03385) model pretrained on the [ILSVRC-2012-CLS](http://www.image-net.org/challenges/LSVRC/2012/) image classification dataset. The decoder is a long short-term memory (LSTM) network. \n\n![alt text](png/model.png)\n\n#### Training phase\nFor the encoder part, the pretrained CNN extracts the feature vector from a given input image. The feature vector is linearly transformed to have the same dimension as the input dimension of the LSTM network. For the decoder part, source and target texts are predefined. For example, if the image description is **\"Giraffes standing next to each other\"**, the source sequence is a list containing **['\\', 'Giraffes', 'standing', 'next', 'to', 'each', 'other']** and the target sequence is a list containing **['Giraffes', 'standing', 'next', 'to', 'each', 'other', '\\']**. Using these source and target sequences and the feature vector, the LSTM decoder is trained as a language model conditioned on the feature vector.\n\n#### Test phase\nIn the test phase, the encoder part is almost same as the training phase. The only difference is that batchnorm layer uses moving average and variance instead of mini-batch statistics. This can be easily implemented using [encoder.eval()](https://github.com/yunjey/pytorch-tutorial/blob/master/tutorials/03-advanced/image_captioning/sample.py#L37). For the decoder part, there is a significant difference between the training phase and the test phase. In the test phase, the LSTM decoder can't see the image description. To deal with this problem, the LSTM decoder feeds back the previosly generated word to the next input. This can be implemented using a [for-loop](https://github.com/yunjey/pytorch-tutorial/blob/master/tutorials/03-advanced/image_captioning/model.py#L48).\n\n\n\n## Usage \n\n\n#### 1. Clone the repositories\n```bash\ngit clone https://github.com/pdollar/coco.git\ncd coco/PythonAPI/\nmake\npython setup.py build\npython setup.py install\ncd ../../\ngit clone https://github.com/yunjey/pytorch-tutorial.git\ncd pytorch-tutorial/tutorials/03-advanced/image_captioning/\n```\n\n#### 2. Download the dataset\n\n```bash\npip install -r requirements.txt\nchmod +x download.sh\n./download.sh\n```\n\n#### 3. Preprocessing\n\n```bash\npython build_vocab.py \npython resize.py\n```\n\n#### 4. Train the model\n\n```bash\npython train.py \n```\n\n#### 5. Test the model \n\n```bash\npython sample.py --image='png/example.png'\n```\n\n
\n\n## Pretrained model\nIf you do not want to train the model from scratch, you can use a pretrained model. You can download the pretrained model [here](https://www.dropbox.com/s/ne0ixz5d58ccbbz/pretrained_model.zip?dl=0) and the vocabulary file [here](https://www.dropbox.com/s/26adb7y9m98uisa/vocap.zip?dl=0). You should extract pretrained_model.zip to `./models/` and vocab.pkl to `./data/` using `unzip` command.\n" }, { "path": "tutorials/03-advanced/image_captioning/build_vocab.py", "content": "import nltk\nimport pickle\nimport argparse\nfrom collections import Counter\nfrom pycocotools.coco import COCO\n\n\nclass Vocabulary(object):\n \"\"\"Simple vocabulary wrapper.\"\"\"\n def __init__(self):\n self.word2idx = {}\n self.idx2word = {}\n self.idx = 0\n\n def add_word(self, word):\n if not word in self.word2idx:\n self.word2idx[word] = self.idx\n self.idx2word[self.idx] = word\n self.idx += 1\n\n def __call__(self, word):\n if not word in self.word2idx:\n return self.word2idx['']\n return self.word2idx[word]\n\n def __len__(self):\n return len(self.word2idx)\n\ndef build_vocab(json, threshold):\n \"\"\"Build a simple vocabulary wrapper.\"\"\"\n coco = COCO(json)\n counter = Counter()\n ids = coco.anns.keys()\n for i, id in enumerate(ids):\n caption = str(coco.anns[id]['caption'])\n tokens = nltk.tokenize.word_tokenize(caption.lower())\n counter.update(tokens)\n\n if (i+1) % 1000 == 0:\n print(\"[{}/{}] Tokenized the captions.\".format(i+1, len(ids)))\n\n # If the word frequency is less than 'threshold', then the word is discarded.\n words = [word for word, cnt in counter.items() if cnt >= threshold]\n\n # Create a vocab wrapper and add some special tokens.\n vocab = Vocabulary()\n vocab.add_word('')\n vocab.add_word('')\n vocab.add_word('')\n vocab.add_word('')\n\n # Add the words to the vocabulary.\n for i, word in enumerate(words):\n vocab.add_word(word)\n return vocab\n\ndef main(args):\n vocab = build_vocab(json=args.caption_path, threshold=args.threshold)\n vocab_path = args.vocab_path\n with open(vocab_path, 'wb') as f:\n pickle.dump(vocab, f)\n print(\"Total vocabulary size: {}\".format(len(vocab)))\n print(\"Saved the vocabulary wrapper to '{}'\".format(vocab_path))\n\n\nif __name__ == '__main__':\n parser = argparse.ArgumentParser()\n parser.add_argument('--caption_path', type=str, \n default='data/annotations/captions_train2014.json', \n help='path for train annotation file')\n parser.add_argument('--vocab_path', type=str, default='./data/vocab.pkl', \n help='path for saving vocabulary wrapper')\n parser.add_argument('--threshold', type=int, default=4, \n help='minimum word count threshold')\n args = parser.parse_args()\n main(args)" }, { "path": "tutorials/03-advanced/image_captioning/data_loader.py", "content": "import torch\nimport torchvision.transforms as transforms\nimport torch.utils.data as data\nimport os\nimport pickle\nimport numpy as np\nimport nltk\nfrom PIL import Image\nfrom build_vocab import Vocabulary\nfrom pycocotools.coco import COCO\n\n\nclass CocoDataset(data.Dataset):\n \"\"\"COCO Custom Dataset compatible with torch.utils.data.DataLoader.\"\"\"\n def __init__(self, root, json, vocab, transform=None):\n \"\"\"Set the path for images, captions and vocabulary wrapper.\n \n Args:\n root: image directory.\n json: coco annotation file path.\n vocab: vocabulary wrapper.\n transform: image transformer.\n \"\"\"\n self.root = root\n self.coco = COCO(json)\n self.ids = list(self.coco.anns.keys())\n self.vocab = vocab\n self.transform = transform\n\n def __getitem__(self, index):\n \"\"\"Returns one data pair (image and caption).\"\"\"\n coco = self.coco\n vocab = self.vocab\n ann_id = self.ids[index]\n caption = coco.anns[ann_id]['caption']\n img_id = coco.anns[ann_id]['image_id']\n path = coco.loadImgs(img_id)[0]['file_name']\n\n image = Image.open(os.path.join(self.root, path)).convert('RGB')\n if self.transform is not None:\n image = self.transform(image)\n\n # Convert caption (string) to word ids.\n tokens = nltk.tokenize.word_tokenize(str(caption).lower())\n caption = []\n caption.append(vocab(''))\n caption.extend([vocab(token) for token in tokens])\n caption.append(vocab(''))\n target = torch.Tensor(caption)\n return image, target\n\n def __len__(self):\n return len(self.ids)\n\n\ndef collate_fn(data):\n \"\"\"Creates mini-batch tensors from the list of tuples (image, caption).\n \n We should build custom collate_fn rather than using default collate_fn, \n because merging caption (including padding) is not supported in default.\n\n Args:\n data: list of tuple (image, caption). \n - image: torch tensor of shape (3, 256, 256).\n - caption: torch tensor of shape (?); variable length.\n\n Returns:\n images: torch tensor of shape (batch_size, 3, 256, 256).\n targets: torch tensor of shape (batch_size, padded_length).\n lengths: list; valid length for each padded caption.\n \"\"\"\n # Sort a data list by caption length (descending order).\n data.sort(key=lambda x: len(x[1]), reverse=True)\n images, captions = zip(*data)\n\n # Merge images (from tuple of 3D tensor to 4D tensor).\n images = torch.stack(images, 0)\n\n # Merge captions (from tuple of 1D tensor to 2D tensor).\n lengths = [len(cap) for cap in captions]\n targets = torch.zeros(len(captions), max(lengths)).long()\n for i, cap in enumerate(captions):\n end = lengths[i]\n targets[i, :end] = cap[:end] \n return images, targets, lengths\n\ndef get_loader(root, json, vocab, transform, batch_size, shuffle, num_workers):\n \"\"\"Returns torch.utils.data.DataLoader for custom coco dataset.\"\"\"\n # COCO caption dataset\n coco = CocoDataset(root=root,\n json=json,\n vocab=vocab,\n transform=transform)\n \n # Data loader for COCO dataset\n # This will return (images, captions, lengths) for each iteration.\n # images: a tensor of shape (batch_size, 3, 224, 224).\n # captions: a tensor of shape (batch_size, padded_length).\n # lengths: a list indicating valid length for each caption. length is (batch_size).\n data_loader = torch.utils.data.DataLoader(dataset=coco, \n batch_size=batch_size,\n shuffle=shuffle,\n num_workers=num_workers,\n collate_fn=collate_fn)\n return data_loader" }, { "path": "tutorials/03-advanced/image_captioning/download.sh", "content": "mkdir data\nwget http://msvocds.blob.core.windows.net/annotations-1-0-3/captions_train-val2014.zip -P ./data/\nwget http://images.cocodataset.org/zips/train2014.zip -P ./data/\nwget http://images.cocodataset.org/zips/val2014.zip -P ./data/\n\nunzip ./data/captions_train-val2014.zip -d ./data/\nrm ./data/captions_train-val2014.zip\nunzip ./data/train2014.zip -d ./data/\nrm ./data/train2014.zip \nunzip ./data/val2014.zip -d ./data/ \nrm ./data/val2014.zip \n" }, { "path": "tutorials/03-advanced/image_captioning/model.py", "content": "import torch\nimport torch.nn as nn\nimport torchvision.models as models\nfrom torch.nn.utils.rnn import pack_padded_sequence\n\n\nclass EncoderCNN(nn.Module):\n def __init__(self, embed_size):\n \"\"\"Load the pretrained ResNet-152 and replace top fc layer.\"\"\"\n super(EncoderCNN, self).__init__()\n resnet = models.resnet152(pretrained=True)\n modules = list(resnet.children())[:-1] # delete the last fc layer.\n self.resnet = nn.Sequential(*modules)\n self.linear = nn.Linear(resnet.fc.in_features, embed_size)\n self.bn = nn.BatchNorm1d(embed_size, momentum=0.01)\n \n def forward(self, images):\n \"\"\"Extract feature vectors from input images.\"\"\"\n with torch.no_grad():\n features = self.resnet(images)\n features = features.reshape(features.size(0), -1)\n features = self.bn(self.linear(features))\n return features\n\n\nclass DecoderRNN(nn.Module):\n def __init__(self, embed_size, hidden_size, vocab_size, num_layers, max_seq_length=20):\n \"\"\"Set the hyper-parameters and build the layers.\"\"\"\n super(DecoderRNN, self).__init__()\n self.embed = nn.Embedding(vocab_size, embed_size)\n self.lstm = nn.LSTM(embed_size, hidden_size, num_layers, batch_first=True)\n self.linear = nn.Linear(hidden_size, vocab_size)\n self.max_seg_length = max_seq_length\n \n def forward(self, features, captions, lengths):\n \"\"\"Decode image feature vectors and generates captions.\"\"\"\n embeddings = self.embed(captions)\n embeddings = torch.cat((features.unsqueeze(1), embeddings), 1)\n packed = pack_padded_sequence(embeddings, lengths, batch_first=True) \n hiddens, _ = self.lstm(packed)\n outputs = self.linear(hiddens[0])\n return outputs\n \n def sample(self, features, states=None):\n \"\"\"Generate captions for given image features using greedy search.\"\"\"\n sampled_ids = []\n inputs = features.unsqueeze(1)\n for i in range(self.max_seg_length):\n hiddens, states = self.lstm(inputs, states) # hiddens: (batch_size, 1, hidden_size)\n outputs = self.linear(hiddens.squeeze(1)) # outputs: (batch_size, vocab_size)\n _, predicted = outputs.max(1) # predicted: (batch_size)\n sampled_ids.append(predicted)\n inputs = self.embed(predicted) # inputs: (batch_size, embed_size)\n inputs = inputs.unsqueeze(1) # inputs: (batch_size, 1, embed_size)\n sampled_ids = torch.stack(sampled_ids, 1) # sampled_ids: (batch_size, max_seq_length)\n return sampled_ids" }, { "path": "tutorials/03-advanced/image_captioning/requirements.txt", "content": "matplotlib\nnltk\nnumpy\nPillow\nargparse" }, { "path": "tutorials/03-advanced/image_captioning/resize.py", "content": "import argparse\nimport os\nfrom PIL import Image\n\n\ndef resize_image(image, size):\n \"\"\"Resize an image to the given size.\"\"\"\n return image.resize(size, Image.ANTIALIAS)\n\ndef resize_images(image_dir, output_dir, size):\n \"\"\"Resize the images in 'image_dir' and save into 'output_dir'.\"\"\"\n if not os.path.exists(output_dir):\n os.makedirs(output_dir)\n\n images = os.listdir(image_dir)\n num_images = len(images)\n for i, image in enumerate(images):\n with open(os.path.join(image_dir, image), 'r+b') as f:\n with Image.open(f) as img:\n img = resize_image(img, size)\n img.save(os.path.join(output_dir, image), img.format)\n if (i+1) % 100 == 0:\n print (\"[{}/{}] Resized the images and saved into '{}'.\"\n .format(i+1, num_images, output_dir))\n\ndef main(args):\n image_dir = args.image_dir\n output_dir = args.output_dir\n image_size = [args.image_size, args.image_size]\n resize_images(image_dir, output_dir, image_size)\n\n\nif __name__ == '__main__':\n parser = argparse.ArgumentParser()\n parser.add_argument('--image_dir', type=str, default='./data/train2014/',\n help='directory for train images')\n parser.add_argument('--output_dir', type=str, default='./data/resized2014/',\n help='directory for saving resized images')\n parser.add_argument('--image_size', type=int, default=256,\n help='size for image after processing')\n args = parser.parse_args()\n main(args)" }, { "path": "tutorials/03-advanced/image_captioning/sample.py", "content": "import torch\nimport matplotlib.pyplot as plt\nimport numpy as np \nimport argparse\nimport pickle \nimport os\nfrom torchvision import transforms \nfrom build_vocab import Vocabulary\nfrom model import EncoderCNN, DecoderRNN\nfrom PIL import Image\n\n\n# Device configuration\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\ndef load_image(image_path, transform=None):\n image = Image.open(image_path).convert('RGB')\n image = image.resize([224, 224], Image.LANCZOS)\n \n if transform is not None:\n image = transform(image).unsqueeze(0)\n \n return image\n\ndef main(args):\n # Image preprocessing\n transform = transforms.Compose([\n transforms.ToTensor(), \n transforms.Normalize((0.485, 0.456, 0.406), \n (0.229, 0.224, 0.225))])\n \n # Load vocabulary wrapper\n with open(args.vocab_path, 'rb') as f:\n vocab = pickle.load(f)\n\n # Build models\n encoder = EncoderCNN(args.embed_size).eval() # eval mode (batchnorm uses moving mean/variance)\n decoder = DecoderRNN(args.embed_size, args.hidden_size, len(vocab), args.num_layers)\n encoder = encoder.to(device)\n decoder = decoder.to(device)\n\n # Load the trained model parameters\n encoder.load_state_dict(torch.load(args.encoder_path))\n decoder.load_state_dict(torch.load(args.decoder_path))\n\n # Prepare an image\n image = load_image(args.image, transform)\n image_tensor = image.to(device)\n \n # Generate an caption from the image\n feature = encoder(image_tensor)\n sampled_ids = decoder.sample(feature)\n sampled_ids = sampled_ids[0].cpu().numpy() # (1, max_seq_length) -> (max_seq_length)\n \n # Convert word_ids to words\n sampled_caption = []\n for word_id in sampled_ids:\n word = vocab.idx2word[word_id]\n sampled_caption.append(word)\n if word == '':\n break\n sentence = ' '.join(sampled_caption)\n \n # Print out the image and the generated caption\n print (sentence)\n image = Image.open(args.image)\n plt.imshow(np.asarray(image))\n \nif __name__ == '__main__':\n parser = argparse.ArgumentParser()\n parser.add_argument('--image', type=str, required=True, help='input image for generating caption')\n parser.add_argument('--encoder_path', type=str, default='models/encoder-5-3000.pkl', help='path for trained encoder')\n parser.add_argument('--decoder_path', type=str, default='models/decoder-5-3000.pkl', help='path for trained decoder')\n parser.add_argument('--vocab_path', type=str, default='data/vocab.pkl', help='path for vocabulary wrapper')\n \n # Model parameters (should be same as paramters in train.py)\n parser.add_argument('--embed_size', type=int , default=256, help='dimension of word embedding vectors')\n parser.add_argument('--hidden_size', type=int , default=512, help='dimension of lstm hidden states')\n parser.add_argument('--num_layers', type=int , default=1, help='number of layers in lstm')\n args = parser.parse_args()\n main(args)\n" }, { "path": "tutorials/03-advanced/image_captioning/train.py", "content": "import argparse\nimport torch\nimport torch.nn as nn\nimport numpy as np\nimport os\nimport pickle\nfrom data_loader import get_loader \nfrom build_vocab import Vocabulary\nfrom model import EncoderCNN, DecoderRNN\nfrom torch.nn.utils.rnn import pack_padded_sequence\nfrom torchvision import transforms\n\n\n# Device configuration\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\ndef main(args):\n # Create model directory\n if not os.path.exists(args.model_path):\n os.makedirs(args.model_path)\n \n # Image preprocessing, normalization for the pretrained resnet\n transform = transforms.Compose([ \n transforms.RandomCrop(args.crop_size),\n transforms.RandomHorizontalFlip(), \n transforms.ToTensor(), \n transforms.Normalize((0.485, 0.456, 0.406), \n (0.229, 0.224, 0.225))])\n \n # Load vocabulary wrapper\n with open(args.vocab_path, 'rb') as f:\n vocab = pickle.load(f)\n \n # Build data loader\n data_loader = get_loader(args.image_dir, args.caption_path, vocab, \n transform, args.batch_size,\n shuffle=True, num_workers=args.num_workers) \n\n # Build the models\n encoder = EncoderCNN(args.embed_size).to(device)\n decoder = DecoderRNN(args.embed_size, args.hidden_size, len(vocab), args.num_layers).to(device)\n \n # Loss and optimizer\n criterion = nn.CrossEntropyLoss()\n params = list(decoder.parameters()) + list(encoder.linear.parameters()) + list(encoder.bn.parameters())\n optimizer = torch.optim.Adam(params, lr=args.learning_rate)\n \n # Train the models\n total_step = len(data_loader)\n for epoch in range(args.num_epochs):\n for i, (images, captions, lengths) in enumerate(data_loader):\n \n # Set mini-batch dataset\n images = images.to(device)\n captions = captions.to(device)\n targets = pack_padded_sequence(captions, lengths, batch_first=True)[0]\n \n # Forward, backward and optimize\n features = encoder(images)\n outputs = decoder(features, captions, lengths)\n loss = criterion(outputs, targets)\n decoder.zero_grad()\n encoder.zero_grad()\n loss.backward()\n optimizer.step()\n\n # Print log info\n if i % args.log_step == 0:\n print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}, Perplexity: {:5.4f}'\n .format(epoch, args.num_epochs, i, total_step, loss.item(), np.exp(loss.item()))) \n \n # Save the model checkpoints\n if (i+1) % args.save_step == 0:\n torch.save(decoder.state_dict(), os.path.join(\n args.model_path, 'decoder-{}-{}.ckpt'.format(epoch+1, i+1)))\n torch.save(encoder.state_dict(), os.path.join(\n args.model_path, 'encoder-{}-{}.ckpt'.format(epoch+1, i+1)))\n\n\nif __name__ == '__main__':\n parser = argparse.ArgumentParser()\n parser.add_argument('--model_path', type=str, default='models/' , help='path for saving trained models')\n parser.add_argument('--crop_size', type=int, default=224 , help='size for randomly cropping images')\n parser.add_argument('--vocab_path', type=str, default='data/vocab.pkl', help='path for vocabulary wrapper')\n parser.add_argument('--image_dir', type=str, default='data/resized2014', help='directory for resized images')\n parser.add_argument('--caption_path', type=str, default='data/annotations/captions_train2014.json', help='path for train annotation json file')\n parser.add_argument('--log_step', type=int , default=10, help='step size for prining log info')\n parser.add_argument('--save_step', type=int , default=1000, help='step size for saving trained models')\n \n # Model parameters\n parser.add_argument('--embed_size', type=int , default=256, help='dimension of word embedding vectors')\n parser.add_argument('--hidden_size', type=int , default=512, help='dimension of lstm hidden states')\n parser.add_argument('--num_layers', type=int , default=1, help='number of layers in lstm')\n \n parser.add_argument('--num_epochs', type=int, default=5)\n parser.add_argument('--batch_size', type=int, default=128)\n parser.add_argument('--num_workers', type=int, default=2)\n parser.add_argument('--learning_rate', type=float, default=0.001)\n args = parser.parse_args()\n print(args)\n main(args)" }, { "path": "tutorials/03-advanced/neural_style_transfer/README.md", "content": "# Neural Style Transfer\n\n[Neural style transfer](https://arxiv.org/abs/1508.06576) is an algorithm that combines the content of one image with the style of another image using CNN. Given a content image and a style image, the goal is to generate a target image that minimizes the content difference with the content image and the style difference with the style image. \n\n

\n\n\n#### Content loss\n\nTo minimize the content difference, we forward propagate the content image and the target image to pretrained [VGGNet](https://arxiv.org/abs/1409.1556) respectively, and extract feature maps from multiple convolutional layers. Then, the target image is updated to minimize the [mean-squared error](https://github.com/yunjey/pytorch-tutorial/blob/master/tutorials/03-advanced/neural_style_transfer/main.py#L81-L82) between the feature maps of the content image and its feature maps. \n\n#### Style loss\n\nAs in computing the content loss, we forward propagate the style image and the target image to the VGGNet and extract convolutional feature maps. To generate a texture that matches the style of the style image, we update the target image by minimizing the mean-squared error between the Gram matrix of the style image and the Gram matrix of the target image (feature correlation minimization). See [here](https://github.com/yunjey/pytorch-tutorial/blob/master/tutorials/03-advanced/neural_style_transfer/main.py#L84-L94) for how to compute the style loss.\n\n\n\n\n
\n\n## Usage \n\n```bash\n$ pip install -r requirements.txt\n$ python main.py --content='png/content.png' --style='png/style.png'\n```\n\n
\n\n## Results\nThe following is the result of applying variaous styles of artwork to Anne Hathaway's photograph.\n\n![alt text](png/neural_style.png)\n" }, { "path": "tutorials/03-advanced/neural_style_transfer/main.py", "content": "from __future__ import division\nfrom torchvision import models\nfrom torchvision import transforms\nfrom PIL import Image\nimport argparse\nimport torch\nimport torchvision\nimport torch.nn as nn\nimport numpy as np\n\n\n# Device configuration\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\ndef load_image(image_path, transform=None, max_size=None, shape=None):\n \"\"\"Load an image and convert it to a torch tensor.\"\"\"\n image = Image.open(image_path)\n \n if max_size:\n scale = max_size / max(image.size)\n size = np.array(image.size) * scale\n image = image.resize(size.astype(int), Image.ANTIALIAS)\n \n if shape:\n image = image.resize(shape, Image.LANCZOS)\n \n if transform:\n image = transform(image).unsqueeze(0)\n \n return image.to(device)\n\n\nclass VGGNet(nn.Module):\n def __init__(self):\n \"\"\"Select conv1_1 ~ conv5_1 activation maps.\"\"\"\n super(VGGNet, self).__init__()\n self.select = ['0', '5', '10', '19', '28'] \n self.vgg = models.vgg19(pretrained=True).features\n \n def forward(self, x):\n \"\"\"Extract multiple convolutional feature maps.\"\"\"\n features = []\n for name, layer in self.vgg._modules.items():\n x = layer(x)\n if name in self.select:\n features.append(x)\n return features\n\n\ndef main(config):\n \n # Image preprocessing\n # VGGNet was trained on ImageNet where images are normalized by mean=[0.485, 0.456, 0.406] and std=[0.229, 0.224, 0.225].\n # We use the same normalization statistics here.\n transform = transforms.Compose([\n transforms.ToTensor(),\n transforms.Normalize(mean=(0.485, 0.456, 0.406), \n std=(0.229, 0.224, 0.225))])\n \n # Load content and style images\n # Make the style image same size as the content image\n content = load_image(config.content, transform, max_size=config.max_size)\n style = load_image(config.style, transform, shape=[content.size(2), content.size(3)])\n \n # Initialize a target image with the content image\n target = content.clone().requires_grad_(True)\n \n optimizer = torch.optim.Adam([target], lr=config.lr, betas=[0.5, 0.999])\n vgg = VGGNet().to(device).eval()\n \n for step in range(config.total_step):\n \n # Extract multiple(5) conv feature vectors\n target_features = vgg(target)\n content_features = vgg(content)\n style_features = vgg(style)\n\n style_loss = 0\n content_loss = 0\n for f1, f2, f3 in zip(target_features, content_features, style_features):\n # Compute content loss with target and content images\n content_loss += torch.mean((f1 - f2)**2)\n\n # Reshape convolutional feature maps\n _, c, h, w = f1.size()\n f1 = f1.view(c, h * w)\n f3 = f3.view(c, h * w)\n\n # Compute gram matrix\n f1 = torch.mm(f1, f1.t())\n f3 = torch.mm(f3, f3.t())\n\n # Compute style loss with target and style images\n style_loss += torch.mean((f1 - f3)**2) / (c * h * w) \n \n # Compute total loss, backprop and optimize\n loss = content_loss + config.style_weight * style_loss \n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n\n if (step+1) % config.log_step == 0:\n print ('Step [{}/{}], Content Loss: {:.4f}, Style Loss: {:.4f}' \n .format(step+1, config.total_step, content_loss.item(), style_loss.item()))\n\n if (step+1) % config.sample_step == 0:\n # Save the generated image\n denorm = transforms.Normalize((-2.12, -2.04, -1.80), (4.37, 4.46, 4.44))\n img = target.clone().squeeze()\n img = denorm(img).clamp_(0, 1)\n torchvision.utils.save_image(img, 'output-{}.png'.format(step+1))\n\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser()\n parser.add_argument('--content', type=str, default='png/content.png')\n parser.add_argument('--style', type=str, default='png/style.png')\n parser.add_argument('--max_size', type=int, default=400)\n parser.add_argument('--total_step', type=int, default=2000)\n parser.add_argument('--log_step', type=int, default=10)\n parser.add_argument('--sample_step', type=int, default=500)\n parser.add_argument('--style_weight', type=float, default=100)\n parser.add_argument('--lr', type=float, default=0.003)\n config = parser.parse_args()\n print(config)\n main(config)" }, { "path": "tutorials/03-advanced/neural_style_transfer/requirements.txt", "content": "argparse\ntorch\ntorchvision\nPillow\n" }, { "path": "tutorials/03-advanced/variational_autoencoder/main.py", "content": "import os\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nfrom torchvision import transforms\nfrom torchvision.utils import save_image\n\n\n# Device configuration\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Create a directory if not exists\nsample_dir = 'samples'\nif not os.path.exists(sample_dir):\n os.makedirs(sample_dir)\n\n# Hyper-parameters\nimage_size = 784\nh_dim = 400\nz_dim = 20\nnum_epochs = 15\nbatch_size = 128\nlearning_rate = 1e-3\n\n# MNIST dataset\ndataset = torchvision.datasets.MNIST(root='../../data',\n train=True,\n transform=transforms.ToTensor(),\n download=True)\n\n# Data loader\ndata_loader = torch.utils.data.DataLoader(dataset=dataset,\n batch_size=batch_size, \n shuffle=True)\n\n\n# VAE model\nclass VAE(nn.Module):\n def __init__(self, image_size=784, h_dim=400, z_dim=20):\n super(VAE, self).__init__()\n self.fc1 = nn.Linear(image_size, h_dim)\n self.fc2 = nn.Linear(h_dim, z_dim)\n self.fc3 = nn.Linear(h_dim, z_dim)\n self.fc4 = nn.Linear(z_dim, h_dim)\n self.fc5 = nn.Linear(h_dim, image_size)\n \n def encode(self, x):\n h = F.relu(self.fc1(x))\n return self.fc2(h), self.fc3(h)\n \n def reparameterize(self, mu, log_var):\n std = torch.exp(log_var/2)\n eps = torch.randn_like(std)\n return mu + eps * std\n\n def decode(self, z):\n h = F.relu(self.fc4(z))\n return F.sigmoid(self.fc5(h))\n \n def forward(self, x):\n mu, log_var = self.encode(x)\n z = self.reparameterize(mu, log_var)\n x_reconst = self.decode(z)\n return x_reconst, mu, log_var\n\nmodel = VAE().to(device)\noptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n\n# Start training\nfor epoch in range(num_epochs):\n for i, (x, _) in enumerate(data_loader):\n # Forward pass\n x = x.to(device).view(-1, image_size)\n x_reconst, mu, log_var = model(x)\n \n # Compute reconstruction loss and kl divergence\n # For KL divergence, see Appendix B in VAE paper or http://yunjey47.tistory.com/43\n reconst_loss = F.binary_cross_entropy(x_reconst, x, size_average=False)\n kl_div = - 0.5 * torch.sum(1 + log_var - mu.pow(2) - log_var.exp())\n \n # Backprop and optimize\n loss = reconst_loss + kl_div\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n \n if (i+1) % 10 == 0:\n print (\"Epoch[{}/{}], Step [{}/{}], Reconst Loss: {:.4f}, KL Div: {:.4f}\" \n .format(epoch+1, num_epochs, i+1, len(data_loader), reconst_loss.item(), kl_div.item()))\n \n with torch.no_grad():\n # Save the sampled images\n z = torch.randn(batch_size, z_dim).to(device)\n out = model.decode(z).view(-1, 1, 28, 28)\n save_image(out, os.path.join(sample_dir, 'sampled-{}.png'.format(epoch+1)))\n\n # Save the reconstructed images\n out, _, _ = model(x)\n x_concat = torch.cat([x.view(-1, 1, 28, 28), out.view(-1, 1, 28, 28)], dim=3)\n save_image(x_concat, os.path.join(sample_dir, 'reconst-{}.png'.format(epoch+1)))" }, { "path": "tutorials/04-utils/tensorboard/README.md", "content": "# TensorBoard in PyTorch\n\nIn this tutorial, we implement a MNIST classifier using a simple neural network and visualize the training process using [TensorBoard](https://www.tensorflow.org/get_started/summaries_and_tensorboard). In training phase, we plot the loss and accuracy functions through `scalar_summary` and visualize the training images through `image_summary`. In addition, we visualize the weight and gradient values of the parameters of the neural network using `histogram_summary`. PyTorch code for handling these summary functions can be found [here](https://github.com/yunjey/pytorch-tutorial/blob/master/tutorials/04-utils/tensorboard/main.py#L81-L97).\n\n![alt text](gif/tensorboard.gif)\n\n
\n\n## Usage \n\n#### 1. Install the dependencies\n```bash\n$ pip install -r requirements.txt\n```\n\n#### 2. Train the model\n```bash\n$ python main.py\n```\n\n#### 3. Open the TensorBoard\nTo run the TensorBoard, open a new terminal and run the command below. Then, open http://localhost:6006/ on your web browser.\n```bash\n$ tensorboard --logdir='./logs' --port=6006\n```\n" }, { "path": "tutorials/04-utils/tensorboard/logger.py", "content": "# Code referenced from https://gist.github.com/gyglim/1f8dfb1b5c82627ae3efcfbbadb9f514\nimport tensorflow as tf\nimport numpy as np\nimport scipy.misc \ntry:\n from StringIO import StringIO # Python 2.7\nexcept ImportError:\n from io import BytesIO # Python 3.x\n\n\nclass Logger(object):\n \n def __init__(self, log_dir):\n \"\"\"Create a summary writer logging to log_dir.\"\"\"\n self.writer = tf.summary.FileWriter(log_dir)\n\n def scalar_summary(self, tag, value, step):\n \"\"\"Log a scalar variable.\"\"\"\n summary = tf.Summary(value=[tf.Summary.Value(tag=tag, simple_value=value)])\n self.writer.add_summary(summary, step)\n\n def image_summary(self, tag, images, step):\n \"\"\"Log a list of images.\"\"\"\n\n img_summaries = []\n for i, img in enumerate(images):\n # Write the image to a string\n try:\n s = StringIO()\n except:\n s = BytesIO()\n scipy.misc.toimage(img).save(s, format=\"png\")\n\n # Create an Image object\n img_sum = tf.Summary.Image(encoded_image_string=s.getvalue(),\n height=img.shape[0],\n width=img.shape[1])\n # Create a Summary value\n img_summaries.append(tf.Summary.Value(tag='%s/%d' % (tag, i), image=img_sum))\n\n # Create and write Summary\n summary = tf.Summary(value=img_summaries)\n self.writer.add_summary(summary, step)\n \n def histo_summary(self, tag, values, step, bins=1000):\n \"\"\"Log a histogram of the tensor of values.\"\"\"\n\n # Create a histogram using numpy\n counts, bin_edges = np.histogram(values, bins=bins)\n\n # Fill the fields of the histogram proto\n hist = tf.HistogramProto()\n hist.min = float(np.min(values))\n hist.max = float(np.max(values))\n hist.num = int(np.prod(values.shape))\n hist.sum = float(np.sum(values))\n hist.sum_squares = float(np.sum(values**2))\n\n # Drop the start of the first bin\n bin_edges = bin_edges[1:]\n\n # Add bin edges and counts\n for edge in bin_edges:\n hist.bucket_limit.append(edge)\n for c in counts:\n hist.bucket.append(c)\n\n # Create and write Summary\n summary = tf.Summary(value=[tf.Summary.Value(tag=tag, histo=hist)])\n self.writer.add_summary(summary, step)\n self.writer.flush()" }, { "path": "tutorials/04-utils/tensorboard/main.py", "content": "import torch\nimport torch.nn as nn\nimport torchvision\nfrom torchvision import transforms\nfrom logger import Logger\n\n\n# Device configuration\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# MNIST dataset \ndataset = torchvision.datasets.MNIST(root='../../data', \n train=True, \n transform=transforms.ToTensor(), \n download=True)\n\n# Data loader\ndata_loader = torch.utils.data.DataLoader(dataset=dataset, \n batch_size=100, \n shuffle=True)\n\n\n# Fully connected neural network with one hidden layer\nclass NeuralNet(nn.Module):\n def __init__(self, input_size=784, hidden_size=500, num_classes=10):\n super(NeuralNet, self).__init__()\n self.fc1 = nn.Linear(input_size, hidden_size) \n self.relu = nn.ReLU()\n self.fc2 = nn.Linear(hidden_size, num_classes) \n \n def forward(self, x):\n out = self.fc1(x)\n out = self.relu(out)\n out = self.fc2(out)\n return out\n\nmodel = NeuralNet().to(device)\n\nlogger = Logger('./logs')\n\n# Loss and optimizer\ncriterion = nn.CrossEntropyLoss() \noptimizer = torch.optim.Adam(model.parameters(), lr=0.00001) \n\ndata_iter = iter(data_loader)\niter_per_epoch = len(data_loader)\ntotal_step = 50000\n\n# Start training\nfor step in range(total_step):\n \n # Reset the data_iter\n if (step+1) % iter_per_epoch == 0:\n data_iter = iter(data_loader)\n\n # Fetch images and labels\n images, labels = next(data_iter)\n images, labels = images.view(images.size(0), -1).to(device), labels.to(device)\n \n # Forward pass\n outputs = model(images)\n loss = criterion(outputs, labels)\n \n # Backward and optimize\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n\n # Compute accuracy\n _, argmax = torch.max(outputs, 1)\n accuracy = (labels == argmax.squeeze()).float().mean()\n\n if (step+1) % 100 == 0:\n print ('Step [{}/{}], Loss: {:.4f}, Acc: {:.2f}' \n .format(step+1, total_step, loss.item(), accuracy.item()))\n\n # ================================================================== #\n # Tensorboard Logging #\n # ================================================================== #\n\n # 1. Log scalar values (scalar summary)\n info = { 'loss': loss.item(), 'accuracy': accuracy.item() }\n\n for tag, value in info.items():\n logger.scalar_summary(tag, value, step+1)\n\n # 2. Log values and gradients of the parameters (histogram summary)\n for tag, value in model.named_parameters():\n tag = tag.replace('.', '/')\n logger.histo_summary(tag, value.data.cpu().numpy(), step+1)\n logger.histo_summary(tag+'/grad', value.grad.data.cpu().numpy(), step+1)\n\n # 3. Log training images (image summary)\n info = { 'images': images.view(-1, 28, 28)[:10].cpu().numpy() }\n\n for tag, images in info.items():\n logger.image_summary(tag, images, step+1)" }, { "path": "tutorials/04-utils/tensorboard/requirements.txt", "content": "tensorflow\ntorch\ntorchvision\nscipy\nnumpy\n" } ]