[
  {
    "path": "README.md",
    "content": "# Adversarial Box - Pytorch Adversarial Attack and Training\n\nLuyu Wang and Gavin Ding, Borealis AI\n\n## Motivation?\n[CleverHans](https://github.com/tensorflow/cleverhans) comes in handy for Tensorflow. However, PyTorch does not have the luck at this moment. [Foolbox](https://github.com/bethgelab/foolbox) supports multiple deep learning frameworks, but it lacks many major implementations (e.g., black-box attack, Carlini-Wagner attack, adversarial training). We feel there is a need to write an easy-to-use and versatile library to help our fellow researchers and engineers.\n\n**We have a much more updated version called [AdverTorch](https://github.com/BorealisAI/advertorch). You can find most of the popular attacks there. This repo will not be maintained anymore.**\n\n## Usage\n    from adversarialbox.attacks import FGSMAttack\n    adversary = FGSMAttack(model, epsilon=0.1)\n    X_adv = adversary.perturb(X_i, y_i)\n\n## Examples\n1. MNIST with FGSM ([code](https://github.com/wanglouis49/pytorch-adversarial_box/blob/master/mnist_attack.py))\n2. Adversarial Training on MNIST ([code](https://github.com/wanglouis49/pytorch-adversarial_box/blob/master/mnist_adv_train.py))\n3. MNIST using a black-box attack ([code](https://github.com/wanglouis49/pytorch-adversarial_box/blob/master/mnist_blackbox.py))\n\n## List of supported attacks\n1. FGSM\n2. PGD\n3. Black-box\n"
  },
  {
    "path": "adversarialbox/__init__.py",
    "content": ""
  },
  {
    "path": "adversarialbox/attacks.py",
    "content": "import copy\nimport numpy as np\nfrom collections import Iterable\nfrom scipy.stats import truncnorm\n\nimport torch\nimport torch.nn as nn\n\nfrom adversarialbox.utils import to_var\n\n# --- White-box attacks ---\n\nclass FGSMAttack(object):\n    def __init__(self, model=None, epsilon=None):\n        \"\"\"\n        One step fast gradient sign method\n        \"\"\"\n        self.model = model\n        self.epsilon = epsilon\n        self.loss_fn = nn.CrossEntropyLoss()\n\n    def perturb(self, X_nat, y, epsilons=None):\n        \"\"\"\n        Given examples (X_nat, y), returns their adversarial\n        counterparts with an attack length of epsilon.\n        \"\"\"\n        # Providing epsilons in batch\n        if epsilons is not None:\n            self.epsilon = epsilons\n\n        X = np.copy(X_nat)\n\n        X_var = to_var(torch.from_numpy(X), requires_grad=True)\n        y_var = to_var(torch.LongTensor(y))\n\n        scores = self.model(X_var)\n        loss = self.loss_fn(scores, y_var)\n        loss.backward()\n        grad_sign = X_var.grad.data.cpu().sign().numpy()\n\n        X += self.epsilon * grad_sign\n        X = np.clip(X, 0, 1)\n\n        return X\n\n\nclass LinfPGDAttack(object):\n    def __init__(self, model=None, epsilon=0.3, k=40, a=0.01, \n        random_start=True):\n        \"\"\"\n        Attack parameter initialization. The attack performs k steps of\n        size a, while always staying within epsilon from the initial\n        point.\n        https://github.com/MadryLab/mnist_challenge/blob/master/pgd_attack.py\n        \"\"\"\n        self.model = model\n        self.epsilon = epsilon\n        self.k = k\n        self.a = a\n        self.rand = random_start\n        self.loss_fn = nn.CrossEntropyLoss()\n\n    def perturb(self, X_nat, y):\n        \"\"\"\n        Given examples (X_nat, y), returns adversarial\n        examples within epsilon of X_nat in l_infinity norm.\n        \"\"\"\n        if self.rand:\n            X = X_nat + np.random.uniform(-self.epsilon, self.epsilon,\n                X_nat.shape).astype('float32')\n        else:\n            X = np.copy(X_nat)\n\n        for i in range(self.k):\n            X_var = to_var(torch.from_numpy(X), requires_grad=True)\n            y_var = to_var(torch.LongTensor(y))\n\n            scores = self.model(X_var)\n            loss = self.loss_fn(scores, y_var)\n            loss.backward()\n            grad = X_var.grad.data.cpu().numpy()\n\n            X += self.a * np.sign(grad)\n\n            X = np.clip(X, X_nat - self.epsilon, X_nat + self.epsilon)\n            X = np.clip(X, 0, 1) # ensure valid pixel range\n\n        return X\n\n\n# --- Black-box attacks ---\n\ndef jacobian(model, x, nb_classes=10):\n    \"\"\"\n    This function will return a list of PyTorch gradients\n    \"\"\"\n    list_derivatives = []\n    x_var = to_var(torch.from_numpy(x), requires_grad=True)\n\n    # derivatives for each class\n    for class_ind in range(nb_classes):\n        score = model(x_var)[:, class_ind]\n        score.backward()\n        list_derivatives.append(x_var.grad.data.cpu().numpy())\n        x_var.grad.data.zero_()\n\n    return list_derivatives\n\n\ndef jacobian_augmentation(model, X_sub_prev, Y_sub, lmbda=0.1):\n    \"\"\"\n    Create new numpy array for adversary training data\n    with twice as many components on the first dimension.\n    \"\"\"\n    X_sub = np.vstack([X_sub_prev, X_sub_prev])\n\n    # For each input in the previous' substitute training iteration\n    for ind, x in enumerate(X_sub_prev):\n        grads = jacobian(model, x)\n        # Select gradient corresponding to the label predicted by the oracle\n        grad = grads[Y_sub[ind]]\n\n        # Compute sign matrix\n        grad_val = np.sign(grad)\n\n        # Create new synthetic point in adversary substitute training set\n        X_sub[len(X_sub_prev)+ind] = X_sub[ind] + lmbda * grad_val #???\n\n    # Return augmented training data (needs to be labeled afterwards)\n    return X_sub\n"
  },
  {
    "path": "adversarialbox/train.py",
    "content": "\"\"\"\nAdversarial training\n\"\"\"\n\nimport copy\nimport numpy as np\nfrom collections import Iterable\nfrom scipy.stats import truncnorm\n\nimport torch\nimport torch.nn as nn\n\nfrom adversarialbox.attacks import FGSMAttack, LinfPGDAttack\nfrom adversarialbox.utils import truncated_normal\n\n\n\ndef adv_train(X, y, model, criterion, adversary):\n    \"\"\"\n    Adversarial training. Returns pertubed mini batch.\n    \"\"\"\n\n    # If adversarial training, need a snapshot of \n    # the model at each batch to compute grad, so \n    # as not to mess up with the optimization step\n    model_cp = copy.deepcopy(model)\n    for p in model_cp.parameters():\n        p.requires_grad = False\n    model_cp.eval()\n    \n    adversary.model = model_cp\n\n    X_adv = adversary.perturb(X.numpy(), y)\n\n    return torch.from_numpy(X_adv)\n\n\ndef FGSM_train_rnd(X, y, model, criterion, fgsm_adversary, epsilon_max=0.3):\n    \"\"\"\n    FGSM with epsilon sampled from a truncated normal distribution.\n    Returns pertubed mini batch.\n    Kurakin et al, ADVERSARIAL MACHINE LEARNING AT SCALE, 2016\n    \"\"\"\n\n    # If adversarial training, need a snapshot of \n    # the model at each batch to compute grad, so \n    # as not to mess up with the optimization step\n    model_cp = copy.deepcopy(model)\n    for p in model_cp.parameters():\n        p.requires_grad = False\n    model_cp.eval()\n    \n    fgsm_adversary.model = model_cp\n\n    # truncated Gaussian\n    m = X.size()[0] # mini-batch size\n    mean, std = 0., epsilon_max/2\n    epsilons = np.abs(truncated_normal(mean, std, m))[:, np.newaxis, \\\n        np.newaxis, np.newaxis]\n\n    X_adv = fgsm_adversary.perturb(X.numpy(), y, epsilons)\n\n    return torch.from_numpy(X_adv)\n\n\n"
  },
  {
    "path": "adversarialbox/utils.py",
    "content": "import numpy as np\nimport torch\nfrom torch.autograd import Variable\nimport torch.nn as nn\nfrom torch.utils.data import sampler\n\n\ndef truncated_normal(mean=0.0, stddev=1.0, m=1):\n    '''\n    The generated values follow a normal distribution with specified \n    mean and standard deviation, except that values whose magnitude is \n    more than 2 standard deviations from the mean are dropped and \n    re-picked. Returns a vector of length m\n    '''\n    samples = []\n    for i in range(m):\n        while True:\n            sample = np.random.normal(mean, stddev)\n            if np.abs(sample) <= 2 * stddev:\n                break\n        samples.append(sample)\n    assert len(samples) == m, \"something wrong\"\n    if m == 1:\n        return samples[0]\n    else:\n        return np.array(samples)\n\n\n# --- PyTorch helpers ---\n\ndef to_var(x, requires_grad=False, volatile=False):\n    \"\"\"\n    Varialbe type that automatically choose cpu or cuda\n    \"\"\"\n    if torch.cuda.is_available():\n        x = x.cuda()\n    return Variable(x, requires_grad=requires_grad, volatile=volatile)\n\n\ndef pred_batch(x, model):\n    \"\"\"\n    batch prediction helper\n    \"\"\"\n    y_pred = np.argmax(model(to_var(x)).data.cpu().numpy(), axis=1)\n    return torch.from_numpy(y_pred)\n\n\ndef test(model, loader, blackbox=False, hold_out_size=None):\n    \"\"\"\n    Check model accuracy on model based on loader (train or test)\n    \"\"\"\n    model.eval()\n\n    num_correct, num_samples = 0, len(loader.dataset)\n\n    if blackbox:\n        num_samples -= hold_out_size\n\n    for x, y in loader:\n        x_var = to_var(x, volatile=True)\n        scores = model(x_var)\n        _, preds = scores.data.cpu().max(1)\n        num_correct += (preds == y).sum()\n\n    acc = float(num_correct)/float(num_samples)\n    print('Got %d/%d correct (%.2f%%) on the clean data' \n        % (num_correct, num_samples, 100 * acc))\n\n    return acc\n\n\ndef attack_over_test_data(model, adversary, param, loader_test, oracle=None):\n    \"\"\"\n    Given target model computes accuracy on perturbed data\n    \"\"\"\n    total_correct = 0\n    total_samples = len(loader_test.dataset)\n\n    # For black-box\n    if oracle is not None:\n        total_samples -= param['hold_out_size']\n\n    for t, (X, y) in enumerate(loader_test):\n        y_pred = pred_batch(X, model)\n        X_adv = adversary.perturb(X.numpy(), y_pred)\n        X_adv = torch.from_numpy(X_adv)\n\n        if oracle is not None:\n            y_pred_adv = pred_batch(X_adv, oracle)\n        else:\n            y_pred_adv = pred_batch(X_adv, model)\n        \n        total_correct += (y_pred_adv.numpy() == y.numpy()).sum()\n\n    acc = total_correct/total_samples\n\n    print('Got %d/%d correct (%.2f%%) on the perturbed data' \n        % (total_correct, total_samples, 100 * acc))\n\n    return acc\n\n\ndef batch_indices(batch_nb, data_length, batch_size):\n    \"\"\"\n    This helper function computes a batch start and end index\n    :param batch_nb: the batch number\n    :param data_length: the total length of the data being parsed by batches\n    :param batch_size: the number of inputs in each batch\n    :return: pair of (start, end) indices\n    \"\"\"\n    # Batch start and end index\n    start = int(batch_nb * batch_size)\n    end = int((batch_nb + 1) * batch_size)\n\n    # When there are not enough inputs left, we reuse some to complete the\n    # batch\n    if end > data_length:\n        shift = end - data_length\n        start -= shift\n        end -= shift\n\n    return start, end\n"
  },
  {
    "path": "mnist_adv_train.py",
    "content": "\"\"\"\nAdversarially train LeNet-5\n\"\"\"\n\nimport torch\nimport torch.nn as nn\nimport torchvision.datasets as datasets\nimport torchvision.transforms as transforms\nfrom torch.autograd import Variable\nimport torch.nn.functional as F\n\nfrom adversarialbox.attacks import FGSMAttack, LinfPGDAttack\nfrom adversarialbox.train import adv_train, FGSM_train_rnd\nfrom adversarialbox.utils import to_var, pred_batch, test\n\nfrom models import LeNet5\n\n\n# Hyper-parameters\nparam = {\n    'batch_size': 128,\n    'test_batch_size': 100,\n    'num_epochs': 15,\n    'delay': 10,\n    'learning_rate': 1e-3,\n    'weight_decay': 5e-4,\n}\n\n\n# Data loaders\ntrain_dataset = datasets.MNIST(root='../data/',train=True, download=True, \n    transform=transforms.ToTensor())\nloader_train = torch.utils.data.DataLoader(train_dataset, \n    batch_size=param['batch_size'], shuffle=True)\n\ntest_dataset = datasets.MNIST(root='../data/', train=False, download=True, \n    transform=transforms.ToTensor())\nloader_test = torch.utils.data.DataLoader(test_dataset, \n    batch_size=param['test_batch_size'], shuffle=True)\n\n\n# Setup the model\nnet = LeNet5()\n\nif torch.cuda.is_available():\n    print('CUDA ensabled.')\n    net.cuda()\nnet.train()\n\n# Adversarial training setup\n#adversary = FGSMAttack(epsilon=0.3)\nadversary = LinfPGDAttack()\n\n# Train the model\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.RMSprop(net.parameters(), lr=param['learning_rate'],\n    weight_decay=param['weight_decay'])\n\nfor epoch in range(param['num_epochs']):\n\n    print('Starting epoch %d / %d' % (epoch + 1, param['num_epochs']))\n\n    for t, (x, y) in enumerate(loader_train):\n\n        x_var, y_var = to_var(x), to_var(y.long())\n        loss = criterion(net(x_var), y_var)\n\n        # adversarial training\n        if epoch+1 > param['delay']:\n            # use predicted label to prevent label leaking\n            y_pred = pred_batch(x, net)\n            x_adv = adv_train(x, y_pred, net, criterion, adversary)\n            x_adv_var = to_var(x_adv)\n            loss_adv = criterion(net(x_adv_var), y_var)\n            loss = (loss + loss_adv) / 2\n\n        if (t + 1) % 100 == 0:\n            print('t = %d, loss = %.8f' % (t + 1, loss.data[0]))\n\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n\ntest(net, loader_test)\n\ntorch.save(net.state_dict(), 'models/adv_trained_lenet5.pkl')\n"
  },
  {
    "path": "mnist_attack.py",
    "content": "\"\"\"\nAdversarial attacks on LeNet5\n\"\"\"\nfrom time import time\nimport torch\nimport torch.nn as nn\nimport torchvision.datasets as datasets\nimport torchvision.transforms as transforms\nfrom torch.autograd import Variable\nimport torch.nn.functional as F\n\nfrom adversarialbox.attacks import FGSMAttack, LinfPGDAttack\nfrom adversarialbox.utils import to_var, pred_batch, test, \\\n    attack_over_test_data\n\nfrom models import LeNet5\n\n\n# Hyper-parameters\nparam = {\n    'test_batch_size': 100,\n    'epsilon': 0.3,\n}\n\n\n# Data loaders\ntest_dataset = datasets.MNIST(root='../data/', train=False, download=True,\n    transform=transforms.ToTensor())\nloader_test = torch.utils.data.DataLoader(test_dataset, \n    batch_size=param['test_batch_size'], shuffle=False)\n\n\n# Setup model to be attacked\nnet = LeNet5()\nnet.load_state_dict(torch.load('models/adv_trained_lenet5.pkl'))\n\nif torch.cuda.is_available():\n    print('CUDA ensabled.')\n    net.cuda()\n\nfor p in net.parameters():\n    p.requires_grad = False\nnet.eval()\n\ntest(net, loader_test)\n\n\n# Adversarial attack\nadversary = FGSMAttack(net, param['epsilon'])\n# adversary = LinfPGDAttack(net, random_start=False)\n\n\nt0 = time()\nattack_over_test_data(net, adversary, param, loader_test)\nprint('{}s eclipsed.'.format(time()-t0))\n"
  },
  {
    "path": "mnist_blackbox.py",
    "content": "\"\"\"\nPyTorch Implementation of Papernot's Black-Box Attack\narXiv:1602.02697\n\"\"\"\n\nimport pickle\nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nimport torchvision.datasets as datasets\nimport torchvision.transforms as transforms\nfrom torch.autograd import Variable\nimport torch.nn.functional as F\nfrom torch.utils.data.sampler import SubsetRandomSampler\n\nfrom adversarialbox.attacks import FGSMAttack, LinfPGDAttack, \\\n    jacobian_augmentation\nfrom adversarialbox.utils import to_var, pred_batch, test, \\\n    attack_over_test_data, batch_indices\n\nfrom models import LeNet5, SubstituteModel\n\n\ndef MNIST_bbox_sub(param, loader_hold_out, loader_test):\n    \"\"\"\n    Train a substitute model using Jacobian data augmentation\n    arXiv:1602.02697\n    \"\"\"\n\n    # Setup the substitute\n    net = SubstituteModel()\n\n    if torch.cuda.is_available():\n        print('CUDA ensabled for the substitute.')\n        net.cuda()\n    net.train()\n\n    # Setup the oracle\n    oracle = LeNet5()\n\n    if torch.cuda.is_available():\n        print('CUDA ensabled for the oracle.')\n        oracle.cuda()\n    oracle.load_state_dict(torch.load(param['oracle_name']+'.pkl'))\n    oracle.eval()\n\n\n    # Setup training\n    criterion = nn.CrossEntropyLoss()\n    # Careful optimization is crucial to train a well-representative \n    # substitute. In Tensorflow Adam has some problem:\n    # (https://github.com/tensorflow/cleverhans/issues/183)\n    # But it works fine here in PyTorch (you may try other optimization\n    # methods\n    optimizer = torch.optim.Adam(net.parameters(), lr=param['learning_rate'])\n\n    # Data held out for initial training\n    data_iter = iter(loader_hold_out)\n    X_sub, y_sub = data_iter.next()\n    X_sub, y_sub = X_sub.numpy(), y_sub.numpy()\n\n    # Train the substitute and augment dataset alternatively\n    for rho in range(param['data_aug']):\n        print(\"Substitute training epoch #\"+str(rho))\n        print(\"Training data: \"+str(len(X_sub)))\n\n        rng = np.random.RandomState()\n\n        # model training\n        for epoch in range(param['nb_epochs']):\n\n            print('Starting epoch %d / %d' % (epoch + 1, param['nb_epochs']))\n\n            # Compute number of batches\n            nb_batches = int(np.ceil(float(len(X_sub)) / \n                param['test_batch_size']))\n            assert nb_batches * param['test_batch_size'] >= len(X_sub)\n\n            # Indices to shuffle training set\n            index_shuf = list(range(len(X_sub)))\n            rng.shuffle(index_shuf)\n\n            for batch in range(nb_batches):\n\n                # Compute batch start and end indices\n                start, end = batch_indices(batch, len(X_sub), \n                    param['test_batch_size'])\n\n                x = X_sub[index_shuf[start:end]]\n                y = y_sub[index_shuf[start:end]]\n\n                scores = net(to_var(torch.from_numpy(x)))\n                loss = criterion(scores, to_var(torch.from_numpy(y).long()))\n\n                optimizer.zero_grad()\n                loss.backward()\n                optimizer.step()\n\n            print('loss = %.8f' % (loss.data[0]))\n        test(net, loader_test, blackbox=True, hold_out_size=param['hold_out_size'])\n\n        # If we are not at last substitute training iteration, augment dataset\n        if rho < param['data_aug'] - 1:\n            print(\"Augmenting substitute training data.\")\n            # Perform the Jacobian augmentation\n            X_sub = jacobian_augmentation(net, X_sub, y_sub)\n\n            print(\"Labeling substitute training data.\")\n            # Label the newly generated synthetic points using the black-box\n            scores = oracle(to_var(torch.from_numpy(X_sub)))\n            # Note here that we take the argmax because the adversary\n            # only has access to the label (not the probabilities) output\n            # by the black-box model\n            y_sub = np.argmax(scores.data.cpu().numpy(), axis=1)\n\n\n    torch.save(net.state_dict(), param['oracle_name']+'_sub.pkl')\n\n\n\n\nif __name__ == \"__main__\":\n\n    # Hyper-parameters\n    param = {\n        'hold_out_size': 150,\n        'test_batch_size': 128,\n        'nb_epochs': 10,\n        'learning_rate': 0.001,\n        'data_aug': 6,\n        'oracle_name': 'models/adv_trained_lenet5',\n        'epsilon': 0.3,\n    }\n\n    # Data loaders\n    # We need to hold out 150 data points from the test data\n    # This is a bit tricky in PyTorch\n    # We adopt the way from:\n    # https://github.com/pytorch/pytorch/issues/1106\n    hold_out_data = datasets.MNIST(root='../data/', train=True,\n        download=True, transform=transforms.ToTensor())\n    test_dataset = datasets.MNIST(root='../data/', train=False, \n        download=True, transform=transforms.ToTensor())\n\n    indices = list(range(test_dataset.test_data.size(0)))\n    split = param['hold_out_size']\n    rng = np.random.RandomState()\n    rng.shuffle(indices)\n\n    hold_out_idx, test_idx = indices[:split], indices[split:]\n\n    hold_out_sampler = SubsetRandomSampler(hold_out_idx)\n    test_sampler = SubsetRandomSampler(test_idx)\n\n    loader_hold_out = torch.utils.data.DataLoader(hold_out_data, \n        batch_size=param['hold_out_size'], sampler=hold_out_sampler,\n        shuffle=False)\n    loader_test = torch.utils.data.DataLoader(test_dataset, \n        batch_size=param['test_batch_size'], sampler=test_sampler,\n        shuffle=False)\n\n\n    # Train the substitute\n    MNIST_bbox_sub(param, loader_hold_out, loader_test)\n\n\n    # Setup models\n    net = SubstituteModel()\n    oracle = LeNet5()\n\n    net.load_state_dict(torch.load(param['oracle_name']+'_sub.pkl'))\n    oracle.load_state_dict(torch.load(param['oracle_name']+'.pkl'))\n\n    if torch.cuda.is_available():\n        net.cuda()\n        oracle.cuda()\n        print('CUDA ensabled.')\n\n    for p in net.parameters():\n        p.requires_grad = False\n\n    net.eval()\n    oracle.eval()\n    \n\n    # Setup adversarial attacks\n    adversary = FGSMAttack(net, param['epsilon'])\n\n    print('For the substitute model:')\n    test(net, loader_test, blackbox=True, hold_out_size=param['hold_out_size'])\n\n    # Setup oracle\n    print('For the oracle'+param['oracle_name'])\n    print('agaist blackbox FGSM attacks using gradients from the substitute:')\n    attack_over_test_data(net, adversary, param, loader_test, oracle)\n"
  },
  {
    "path": "models.py",
    "content": "import torch\nimport torch.nn as nn\n\n\nclass LeNet5(nn.Module):\n    def __init__(self):\n        super(LeNet5, self).__init__()\n        self.conv1 = nn.Conv2d(1, 32, kernel_size=3, padding=1, stride=1)\n        self.relu1 = nn.ReLU(inplace=True)\n        self.maxpool1 = nn.MaxPool2d(2)\n        self.conv2 = nn.Conv2d(32, 64, kernel_size=3, padding=1, stride=1)\n        self.relu2 = nn.ReLU(inplace=True)\n        self.maxpool2 = nn.MaxPool2d(2)\n        self.linear1 = nn.Linear(7*7*64, 200)\n        self.relu3 = nn.ReLU(inplace=True)\n        self.linear2 = nn.Linear(200, 10)\n\n    def forward(self, x):\n        out = self.maxpool1(self.relu1(self.conv1(x)))\n        out = self.maxpool2(self.relu2(self.conv2(out)))\n        out = out.view(out.size(0), -1)\n        out = self.relu3(self.linear1(out))\n        out = self.linear2(out)\n        return out\n\n\nclass SubstituteModel(nn.Module):\n\n    def __init__(self):\n        super(SubstituteModel, self).__init__()\n        self.linear1 = nn.Linear(28*28, 200)\n        self.relu1 = nn.ReLU(inplace=True)\n        self.linear2 = nn.Linear(200, 200)\n        self.relu2 = nn.ReLU(inplace=True)\n        self.linear3 = nn.Linear(200, 10)\n\n    def forward(self, x):\n        out = x.view(x.size(0), -1)\n        out = self.relu1(self.linear1(out))\n        out = self.relu2(self.linear2(out))\n        out = self.linear3(out)\n        return out\n"
  }
]