[
  {
    "path": ".gitignore",
    "content": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nenv/\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\n*.egg-info/\n.installed.cfg\n*.egg\n\n# PyInstaller\n#  Usually these files are written by a python script from a template\n#  before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*,cover\n.hypothesis/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\ntarget/\n\n# IPython Notebook\n.ipynb_checkpoints\n\n# pyenv\n.python-version\n\n# celery beat schedule file\ncelerybeat-schedule\n\n# dotenv\n.env\n\n# virtualenv\nvenv/\nENV/\n\n# Spyder project settings\n.spyderproject\n\n# Rope project settings\n.ropeproject\n\n*.sublime*\nMNIST_data/\nGAN/out/\nVAE/out/\nout/\n\n# Unreleased\nGAN/unrolled_gan/\nGAN/loss_sensitive_gan/\nGAN/generative_adversarial_parallelization/gap_tensorflow.py\n\n.vscode\n"
  },
  {
    "path": "GAN/ali_bigan/ali_bigan_pytorch.py",
    "content": "import torch\nimport torch.nn\nimport torch.nn.functional as nn\nimport torch.autograd as autograd\nimport torch.optim as optim\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\nfrom itertools import *\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 32\nz_dim = 10\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\ncnt = 0\nlr = 1e-3\n\n\ndef log(x):\n    return torch.log(x + 1e-8)\n\n\n# Inference net (Encoder) Q(z|X)\nQ = torch.nn.Sequential(\n    torch.nn.Linear(X_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, z_dim)\n)\n\n# Generator net (Decoder) P(X|z)\nP = torch.nn.Sequential(\n    torch.nn.Linear(z_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, X_dim),\n    torch.nn.Sigmoid()\n)\n\nD_ = torch.nn.Sequential(\n    torch.nn.Linear(X_dim + z_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, 1),\n    torch.nn.Sigmoid()\n)\n\n\ndef D(X, z):\n    return D_(torch.cat([X, z], 1))\n\n\ndef reset_grad():\n    Q.zero_grad()\n    P.zero_grad()\n    D_.zero_grad()\n\n\nG_solver = optim.Adam(chain(Q.parameters(), P.parameters()), lr=lr)\nD_solver = optim.Adam(D_.parameters(), lr=lr)\n\n\nfor it in range(1000000):\n    # Sample data\n    z = Variable(torch.randn(mb_size, z_dim))\n    X, _ = mnist.train.next_batch(mb_size)\n    X = Variable(torch.from_numpy(X))\n\n    # Discriminator\n    z_hat = Q(X)\n    X_hat = P(z)\n\n    D_enc = D(X, z_hat)\n    D_gen = D(X_hat, z)\n\n    D_loss = -torch.mean(log(D_enc) + log(1 - D_gen))\n\n    D_loss.backward()\n    D_solver.step()\n    G_solver.step()\n    reset_grad()\n\n    # Autoencoder Q, P\n    z_hat = Q(X)\n    X_hat = P(z)\n\n    D_enc = D(X, z_hat)\n    D_gen = D(X_hat, z)\n\n    G_loss = -torch.mean(log(D_gen) + log(1 - D_enc))\n\n    G_loss.backward()\n    G_solver.step()\n    reset_grad()\n\n    # Print and plot every now and then\n    if it % 1000 == 0:\n        print('Iter-{}; D_loss: {:.4}; G_loss: {:.4}'\n              .format(it, D_loss.data[0], G_loss.data[0]))\n\n        samples = P(z).data.numpy()[:16]\n\n        fig = plt.figure(figsize=(4, 4))\n        gs = gridspec.GridSpec(4, 4)\n        gs.update(wspace=0.05, hspace=0.05)\n\n        for i, sample in enumerate(samples):\n            ax = plt.subplot(gs[i])\n            plt.axis('off')\n            ax.set_xticklabels([])\n            ax.set_yticklabels([])\n            ax.set_aspect('equal')\n            plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n        if not os.path.exists('out/'):\n            os.makedirs('out/')\n\n        plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight')\n        cnt += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/ali_bigan/ali_bigan_tensorflow.py",
    "content": "import tensorflow as tf\nfrom tensorflow.examples.tutorials.mnist import input_data\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\n\n\nmb_size = 32\nX_dim = 784\nz_dim = 64\nh_dim = 128\nlr = 1e-3\nd_steps = 3\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\n\n\ndef plot(samples):\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    return fig\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)\n    return tf.random_normal(shape=size, stddev=xavier_stddev)\n\n\ndef log(x):\n    return tf.log(x + 1e-8)\n\n\nX = tf.placeholder(tf.float32, shape=[None, X_dim])\nz = tf.placeholder(tf.float32, shape=[None, z_dim])\n\nD_W1 = tf.Variable(xavier_init([X_dim + z_dim, h_dim]))\nD_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nD_W2 = tf.Variable(xavier_init([h_dim, 1]))\nD_b2 = tf.Variable(tf.zeros(shape=[1]))\n\nQ_W1 = tf.Variable(xavier_init([X_dim, h_dim]))\nQ_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nQ_W2 = tf.Variable(xavier_init([h_dim, z_dim]))\nQ_b2 = tf.Variable(tf.zeros(shape=[z_dim]))\n\nP_W1 = tf.Variable(xavier_init([z_dim, h_dim]))\nP_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nP_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nP_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\ntheta_G = [Q_W1, Q_W2, Q_b1, Q_b2, P_W1, P_W2, P_b1, P_b2]\ntheta_D = [D_W1, D_W2, D_b1, D_b2]\n\n\ndef sample_z(m, n):\n    return np.random.uniform(-1., 1., size=[m, n])\n\n\ndef Q(X):\n    h = tf.nn.relu(tf.matmul(X, Q_W1) + Q_b1)\n    h = tf.matmul(h, Q_W2) + Q_b2\n    return h\n\n\ndef P(z):\n    h = tf.nn.relu(tf.matmul(z, P_W1) + P_b1)\n    h = tf.matmul(h, P_W2) + P_b2\n    return tf.nn.sigmoid(h)\n\n\ndef D(X, z):\n    inputs = tf.concat([X, z], axis=1)\n    h = tf.nn.relu(tf.matmul(inputs, D_W1) + D_b1)\n    return tf.nn.sigmoid(tf.matmul(h, D_W2) + D_b2)\n\n\nz_hat = Q(X)\nX_hat = P(z)\n\nD_enc = D(X, z_hat)\nD_gen = D(X_hat, z)\n\nD_loss = -tf.reduce_mean(log(D_enc) + log(1 - D_gen))\nG_loss = -tf.reduce_mean(log(D_gen) + log(1 - D_enc))\n\nD_solver = (tf.train.AdamOptimizer(learning_rate=lr)\n            .minimize(D_loss, var_list=theta_D))\nG_solver = (tf.train.AdamOptimizer(learning_rate=lr)\n            .minimize(G_loss, var_list=theta_G))\n\nsess = tf.Session()\nsess.run(tf.global_variables_initializer())\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\ni = 0\n\nfor it in range(1000000):\n    X_mb, _ = mnist.train.next_batch(mb_size)\n    z_mb = sample_z(mb_size, z_dim)\n\n    _, D_loss_curr = sess.run(\n        [D_solver, D_loss], feed_dict={X: X_mb, z: z_mb}\n    )\n\n    _, G_loss_curr = sess.run(\n        [G_solver, G_loss], feed_dict={X: X_mb, z: z_mb}\n    )\n\n    if it % 1000 == 0:\n        print('Iter: {}; D_loss: {:.4}; G_loss: {:.4}'\n              .format(it, D_loss_curr, G_loss_curr))\n\n        samples = sess.run(X_hat, feed_dict={z: sample_z(16, z_dim)})\n\n        fig = plot(samples)\n        plt.savefig('out/{}.png'\n                    .format(str(i).zfill(3)), bbox_inches='tight')\n        i += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/auxiliary_classifier_gan/ac_gan_pytorch.py",
    "content": "import torch\nimport torch.nn.functional as nn\nimport torch.autograd as autograd\nimport torch.optim as optim\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 32\nz_dim = 16\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\ncnt = 0\nlr = 1e-3\neps = 1e-8\n\n\nG_ = torch.nn.Sequential(\n    torch.nn.Linear(z_dim + y_dim, h_dim),\n    torch.nn.PReLU(),\n    torch.nn.Linear(h_dim, X_dim),\n    torch.nn.Sigmoid()\n)\n\n\ndef G(z, c):\n    inputs = torch.cat([z, c], 1)\n    return G_(inputs)\n\n\nD_shared = torch.nn.Sequential(\n    torch.nn.Linear(X_dim, h_dim),\n    torch.nn.PReLU()\n)\n\nD_gan = torch.nn.Sequential(\n    torch.nn.Linear(h_dim, 1),\n    torch.nn.Sigmoid()\n)\n\nD_aux = torch.nn.Sequential(\n    torch.nn.Linear(h_dim, y_dim),\n)\n\n\ndef D(X):\n    h = D_shared(X)\n    return D_gan(h), D_aux(h)\n\n\nnets = [G_, D_shared, D_gan, D_aux]\n\nG_params = G_.parameters()\nD_params = (list(D_shared.parameters()) + list(D_gan.parameters()) +\n            list(D_aux.parameters()))\n\n\ndef reset_grad():\n    for net in nets:\n        net.zero_grad()\n\n\nG_solver = optim.Adam(G_params, lr=lr)\nD_solver = optim.Adam(D_params, lr=lr)\n\n\nfor it in range(100000):\n    # Sample data\n    X, y = mnist.train.next_batch(mb_size)\n    X = Variable(torch.from_numpy(X))\n    # c is one-hot\n    c = Variable(torch.from_numpy(y.astype('float32')))\n    # y_true is not one-hot (requirement from nn.cross_entropy)\n    y_true = Variable(torch.from_numpy(y.argmax(axis=1).astype('int')))\n    # z noise\n    z = Variable(torch.randn(mb_size, z_dim))\n\n    \"\"\" Discriminator \"\"\"\n    G_sample = G(z, c)\n    D_real, C_real = D(X)\n    D_fake, C_fake = D(G_sample)\n\n    # GAN's D loss\n    D_loss = torch.mean(torch.log(D_real + eps) + torch.log(1 - D_fake + eps))\n    # Cross entropy aux loss\n    C_loss = -nn.cross_entropy(C_real, y_true) - nn.cross_entropy(C_fake, y_true)\n\n    # Maximize\n    DC_loss = -(D_loss + C_loss)\n\n    DC_loss.backward()\n    D_solver.step()\n\n    reset_grad()\n\n    \"\"\" Generator \"\"\"\n    G_sample = G(z, c)\n    D_fake, C_fake = D(G_sample)\n    _, C_real = D(X)\n\n    # GAN's G loss\n    G_loss = torch.mean(torch.log(D_fake + eps))\n    # Cross entropy aux loss\n    C_loss = -nn.cross_entropy(C_real, y_true) - nn.cross_entropy(C_fake, y_true)\n\n    # Maximize\n    GC_loss = -(G_loss + C_loss)\n\n    GC_loss.backward()\n    G_solver.step()\n\n    reset_grad()\n\n    # Print and plot every now and then\n    if it % 1000 == 0:\n        idx = np.random.randint(0, 10)\n        c = np.zeros([16, y_dim])\n        c[range(16), idx] = 1\n        c = Variable(torch.from_numpy(c.astype('float32')))\n\n        z = Variable(torch.randn(16, z_dim))\n\n        samples = G(z, c).data.numpy()\n\n        print('Iter-{}; D_loss: {:.4}; G_loss: {:.4}; Idx: {}'\n              .format(it, -D_loss.data[0], -G_loss.data[0], idx))\n\n        fig = plt.figure(figsize=(4, 4))\n        gs = gridspec.GridSpec(4, 4)\n        gs.update(wspace=0.05, hspace=0.05)\n\n        for i, sample in enumerate(samples):\n            ax = plt.subplot(gs[i])\n            plt.axis('off')\n            ax.set_xticklabels([])\n            ax.set_yticklabels([])\n            ax.set_aspect('equal')\n            plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n        if not os.path.exists('out/'):\n            os.makedirs('out/')\n\n        plt.savefig('out/{}.png'\n                    .format(str(cnt).zfill(3)), bbox_inches='tight')\n        cnt += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/auxiliary_classifier_gan/ac_gan_tensorflow.py",
    "content": "import tensorflow as tf\nfrom tensorflow.examples.tutorials.mnist import input_data\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\n\nmb_size = 32\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nz_dim = 10\nh_dim = 128\neps = 1e-8\nlr = 1e-3\nd_steps = 3\n\n\ndef plot(samples):\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    return fig\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)\n    return tf.random_normal(shape=size, stddev=xavier_stddev)\n\n\nX = tf.placeholder(tf.float32, shape=[None, X_dim])\ny = tf.placeholder(tf.float32, shape=[None, y_dim])\nz = tf.placeholder(tf.float32, shape=[None, z_dim])\n\nG_W1 = tf.Variable(xavier_init([z_dim + y_dim, h_dim]))\nG_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nG_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nG_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\n\ndef generator(z, c):\n    inputs = tf.concat(axis=1, values=[z, c])\n    G_h1 = tf.nn.relu(tf.matmul(inputs, G_W1) + G_b1)\n    G_log_prob = tf.matmul(G_h1, G_W2) + G_b2\n    G_prob = tf.nn.sigmoid(G_log_prob)\n    return G_prob\n\n\nD_W1 = tf.Variable(xavier_init([X_dim, h_dim]))\nD_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nD_W2_gan = tf.Variable(xavier_init([h_dim, 1]))\nD_b2_gan = tf.Variable(tf.zeros(shape=[1]))\nD_W2_aux = tf.Variable(xavier_init([h_dim, y_dim]))\nD_b2_aux = tf.Variable(tf.zeros(shape=[y_dim]))\n\n\ndef discriminator(X):\n    D_h1 = tf.nn.relu(tf.matmul(X, D_W1) + D_b1)\n    out_gan = tf.nn.sigmoid(tf.matmul(D_h1, D_W2_gan) + D_b2_gan)\n    out_aux = tf.matmul(D_h1, D_W2_aux) + D_b2_aux\n    return out_gan, out_aux\n\n\ntheta_G = [G_W1, G_W2, G_b1, G_b2]\ntheta_D = [D_W1, D_W2_gan, D_W2_aux, D_b1, D_b2_gan, D_b2_aux]\n\n\ndef sample_z(m, n):\n    return np.random.uniform(-1., 1., size=[m, n])\n\n\ndef cross_entropy(logit, y):\n    return -tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logit, labels=y))\n\n\nG_sample = generator(z, y)\n\nD_real, C_real = discriminator(X)\nD_fake, C_fake = discriminator(G_sample)\n\n# Cross entropy aux loss\nC_loss = cross_entropy(C_real, y) + cross_entropy(C_fake, y)\n\n# GAN D loss\nD_loss = tf.reduce_mean(tf.log(D_real + eps) + tf.log(1. - D_fake + eps))\nDC_loss = -(D_loss + C_loss)\n\n# GAN's G loss\nG_loss = tf.reduce_mean(tf.log(D_fake + eps))\nGC_loss = -(G_loss + C_loss)\n\nD_solver = (tf.train.AdamOptimizer(learning_rate=lr)\n            .minimize(DC_loss, var_list=theta_D))\nG_solver = (tf.train.AdamOptimizer(learning_rate=lr)\n            .minimize(GC_loss, var_list=theta_G))\n\n\nsess = tf.Session()\nsess.run(tf.global_variables_initializer())\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\ni = 0\n\nfor it in range(1000000):\n    X_mb, y_mb = mnist.train.next_batch(mb_size)\n    z_mb = sample_z(mb_size, z_dim)\n\n    _, DC_loss_curr = sess.run(\n        [D_solver, DC_loss],\n        feed_dict={X: X_mb, y: y_mb, z: z_mb}\n    )\n\n    _, GC_loss_curr = sess.run(\n        [G_solver, GC_loss],\n        feed_dict={X: X_mb, y: y_mb, z: z_mb}\n    )\n\n    if it % 1000 == 0:\n        idx = np.random.randint(0, 10)\n        c = np.zeros([16, y_dim])\n        c[range(16), idx] = 1\n\n        samples = sess.run(G_sample, feed_dict={z: sample_z(16, z_dim), y: c})\n\n        print('Iter: {}; DC_loss: {:.4}; GC_loss: {:.4}; Idx; {}'\n              .format(it, DC_loss_curr, GC_loss_curr, idx))\n\n        fig = plot(samples)\n        plt.savefig('out/{}.png'\n                    .format(str(i).zfill(3)), bbox_inches='tight')\n        i += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/boundary_equilibrium_gan/began_pytorch.py",
    "content": "import torch\nimport torch.nn\nimport torch.nn.functional as nn\nimport torch.autograd as autograd\nimport torch.optim as optim\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 32\nz_dim = 10\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\ncnt = 0\nd_step = 3\nlr = 1e-3\nm = 5\nlam = 1e-3\nk = 0\ngamma = 0.5\n\n\nG = torch.nn.Sequential(\n    torch.nn.Linear(z_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, X_dim),\n    torch.nn.Sigmoid()\n)\n\nD_ = torch.nn.Sequential(\n    torch.nn.Linear(X_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, X_dim),\n)\n\n\n# D is an autoencoder, approximating Gaussian\ndef D(X):\n    X_recon = D_(X)\n    # Use Laplace MLE as in the paper\n    return torch.mean(torch.sum(torch.abs(X - X_recon), 1))\n\n\ndef reset_grad():\n    G.zero_grad()\n    D_.zero_grad()\n\n\nG_solver = optim.Adam(G.parameters(), lr=lr)\nD_solver = optim.Adam(D_.parameters(), lr=lr)\n\n\nfor it in range(1000000):\n    # Sample data\n    X, _ = mnist.train.next_batch(mb_size)\n    X = Variable(torch.from_numpy(X))\n\n    # Dicriminator\n    z_D = Variable(torch.randn(mb_size, z_dim))\n\n    D_loss = D(X) - k * D(G(z_D))\n\n    D_loss.backward()\n    D_solver.step()\n    reset_grad()\n\n    # Generator\n    z_G = Variable(torch.randn(mb_size, z_dim))\n\n    G_loss = D(G(z_G))\n\n    G_loss.backward()\n    G_solver.step()\n    reset_grad()\n\n    # Update k, the equlibrium\n    k = k + lam * (gamma*D(X) - D(G(z_G)))\n    k = k.data[0]  # k is variable, so unvariable it so that no gradient prop.\n\n    # Print and plot every now and then\n    if it % 1000 == 0:\n        measure = D(X) + torch.abs(gamma*D(X) - D(G(z_G)))\n\n        print('Iter-{}; Convergence measure: {:.4}'\n              .format(it, measure.data[0]))\n\n        samples = G(z_G).data.numpy()[:16]\n\n        fig = plt.figure(figsize=(4, 4))\n        gs = gridspec.GridSpec(4, 4)\n        gs.update(wspace=0.05, hspace=0.05)\n\n        for i, sample in enumerate(samples):\n            ax = plt.subplot(gs[i])\n            plt.axis('off')\n            ax.set_xticklabels([])\n            ax.set_yticklabels([])\n            ax.set_aspect('equal')\n            plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n        if not os.path.exists('out/'):\n            os.makedirs('out/')\n\n        plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight')\n        cnt += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/boundary_equilibrium_gan/began_tensorflow.py",
    "content": "import tensorflow as tf\nfrom tensorflow.examples.tutorials.mnist import input_data\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\n\n\nmb_size = 32\nX_dim = 784\nz_dim = 64\nh_dim = 128\nlr = 1e-3\nm = 5\nlam = 1e-3\ngamma = 0.5\nk_curr = 0\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\n\n\ndef plot(samples):\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    return fig\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)\n    return tf.random_normal(shape=size, stddev=xavier_stddev)\n\n\nX = tf.placeholder(tf.float32, shape=[None, X_dim])\nz = tf.placeholder(tf.float32, shape=[None, z_dim])\nk = tf.placeholder(tf.float32)\n\nD_W1 = tf.Variable(xavier_init([X_dim, h_dim]))\nD_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nD_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nD_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\nG_W1 = tf.Variable(xavier_init([z_dim, h_dim]))\nG_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nG_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nG_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\ntheta_G = [G_W1, G_W2, G_b1, G_b2]\ntheta_D = [D_W1, D_W2, D_b1, D_b2]\n\n\ndef sample_z(m, n):\n    return np.random.uniform(-1., 1., size=[m, n])\n\n\ndef G(z):\n    G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1)\n    G_log_prob = tf.matmul(G_h1, G_W2) + G_b2\n    G_prob = tf.nn.sigmoid(G_log_prob)\n    return G_prob\n\n\ndef D(X):\n    D_h1 = tf.nn.relu(tf.matmul(X, D_W1) + D_b1)\n    X_recon = tf.matmul(D_h1, D_W2) + D_b2\n    return tf.reduce_mean(tf.reduce_sum((X - X_recon)**2, 1))\n\n\nG_sample = G(z)\n\nD_real = D(X)\nD_fake = D(G_sample)\n\nD_loss = D_real - k*D_fake\nG_loss = D_fake\n\nD_solver = (tf.train.AdamOptimizer(learning_rate=lr)\n            .minimize(D_loss, var_list=theta_D))\nG_solver = (tf.train.AdamOptimizer(learning_rate=lr)\n            .minimize(G_loss, var_list=theta_G))\n\nsess = tf.Session()\nsess.run(tf.global_variables_initializer())\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\ni = 0\n\nfor it in range(1000000):\n    X_mb, _ = mnist.train.next_batch(mb_size)\n\n    _, D_real_curr = sess.run(\n        [D_solver, D_real],\n        feed_dict={X: X_mb, z: sample_z(mb_size, z_dim), k: k_curr}\n    )\n\n    _, D_fake_curr = sess.run(\n        [G_solver, D_fake],\n        feed_dict={X: X_mb, z: sample_z(mb_size, z_dim)}\n    )\n\n    k_curr = k_curr + lam * (gamma*D_real_curr - D_fake_curr)\n\n    if it % 1000 == 0:\n        measure = D_real_curr + np.abs(gamma*D_real_curr - D_fake_curr)\n\n        print('Iter-{}; Convergence measure: {:.4}'\n              .format(it, measure))\n\n        samples = sess.run(G_sample, feed_dict={z: sample_z(16, z_dim)})\n\n        fig = plot(samples)\n        plt.savefig('out/{}.png'\n                    .format(str(i).zfill(3)), bbox_inches='tight')\n        i += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/boundary_seeking_gan/bgan_pytorch.py",
    "content": "import torch\nimport torch.nn\nimport torch.nn.functional as nn\nimport torch.autograd as autograd\nimport torch.optim as optim\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 32\nz_dim = 10\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\ncnt = 0\nlr = 1e-3\n\n\ndef log(x):\n    return torch.log(x + 1e-8)\n\n\nG = torch.nn.Sequential(\n    torch.nn.Linear(z_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, X_dim),\n    torch.nn.Sigmoid()\n)\n\n\nD = torch.nn.Sequential(\n    torch.nn.Linear(X_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, 1),\n    torch.nn.Sigmoid()\n)\n\n\ndef reset_grad():\n    G.zero_grad()\n    D.zero_grad()\n\n\nG_solver = optim.Adam(G.parameters(), lr=lr)\nD_solver = optim.Adam(D.parameters(), lr=lr)\n\n\nfor it in range(1000000):\n    # Sample data\n    z = Variable(torch.randn(mb_size, z_dim))\n    X, _ = mnist.train.next_batch(mb_size)\n    X = Variable(torch.from_numpy(X))\n\n    # Dicriminator\n    G_sample = G(z)\n    D_real = D(X)\n    D_fake = D(G_sample)\n\n    D_loss = -torch.mean(log(D_real) + log(1 - D_fake))\n\n    D_loss.backward()\n    D_solver.step()\n    reset_grad()\n\n    # Generator\n    G_sample = G(z)\n    D_fake = D(G_sample)\n\n    G_loss = 0.5 * torch.mean((log(D_fake) - log(1 - D_fake))**2)\n\n    G_loss.backward()\n    G_solver.step()\n    reset_grad()\n\n    # Print and plot every now and then\n    if it % 1000 == 0:\n        print('Iter-{}; D_loss: {:.4}; G_loss: {:.4}'\n              .format(it, D_loss.data[0], G_loss.data[0]))\n\n        samples = G(z).data.numpy()[:16]\n\n        fig = plt.figure(figsize=(4, 4))\n        gs = gridspec.GridSpec(4, 4)\n        gs.update(wspace=0.05, hspace=0.05)\n\n        for i, sample in enumerate(samples):\n            ax = plt.subplot(gs[i])\n            plt.axis('off')\n            ax.set_xticklabels([])\n            ax.set_yticklabels([])\n            ax.set_aspect('equal')\n            plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n        if not os.path.exists('out/'):\n            os.makedirs('out/')\n\n        plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight')\n        cnt += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/boundary_seeking_gan/bgan_tensorflow.py",
    "content": "import tensorflow as tf\nfrom tensorflow.examples.tutorials.mnist import input_data\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\n\n\nmb_size = 32\nX_dim = 784\nz_dim = 64\nh_dim = 128\nlr = 1e-3\nd_steps = 3\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\n\n\ndef plot(samples):\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    return fig\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)\n    return tf.random_normal(shape=size, stddev=xavier_stddev)\n\n\ndef log(x):\n    return tf.log(x + 1e-8)\n\n\nX = tf.placeholder(tf.float32, shape=[None, X_dim])\nz = tf.placeholder(tf.float32, shape=[None, z_dim])\n\nD_W1 = tf.Variable(xavier_init([X_dim, h_dim]))\nD_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nD_W2 = tf.Variable(xavier_init([h_dim, 1]))\nD_b2 = tf.Variable(tf.zeros(shape=[1]))\n\nG_W1 = tf.Variable(xavier_init([z_dim, h_dim]))\nG_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nG_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nG_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\ntheta_G = [G_W1, G_W2, G_b1, G_b2]\ntheta_D = [D_W1, D_W2, D_b1, D_b2]\n\n\ndef sample_z(m, n):\n    return np.random.uniform(-1., 1., size=[m, n])\n\n\ndef generator(z):\n    G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1)\n    G_log_prob = tf.matmul(G_h1, G_W2) + G_b2\n    G_prob = tf.nn.sigmoid(G_log_prob)\n    return G_prob\n\n\ndef discriminator(x):\n    D_h1 = tf.nn.relu(tf.matmul(x, D_W1) + D_b1)\n    out = tf.nn.sigmoid(tf.matmul(D_h1, D_W2) + D_b2)\n    return out\n\n\nG_sample = generator(z)\n\nD_real = discriminator(X)\nD_fake = discriminator(G_sample)\n\nD_loss = -tf.reduce_mean(log(D_real) + log(1 - D_fake))\nG_loss = 0.5 * tf.reduce_mean((log(D_fake) - log(1 - D_fake))**2)\n\nD_solver = (tf.train.AdamOptimizer(learning_rate=lr)\n            .minimize(D_loss, var_list=theta_D))\nG_solver = (tf.train.AdamOptimizer(learning_rate=lr)\n            .minimize(G_loss, var_list=theta_G))\n\nsess = tf.Session()\nsess.run(tf.global_variables_initializer())\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\ni = 0\n\nfor it in range(1000000):\n    X_mb, _ = mnist.train.next_batch(mb_size)\n    z_mb = sample_z(mb_size, z_dim)\n\n    _, D_loss_curr = sess.run(\n        [D_solver, D_loss],\n        feed_dict={X: X_mb, z: z_mb}\n    )\n\n    _, G_loss_curr = sess.run(\n        [G_solver, G_loss],\n        feed_dict={X: X_mb, z: sample_z(mb_size, z_dim)}\n    )\n\n    if it % 1000 == 0:\n        print('Iter: {}; D_loss: {:.4}; G_loss: {:.4}'\n              .format(it, D_loss_curr, G_loss_curr))\n\n        samples = sess.run(G_sample, feed_dict={z: sample_z(16, z_dim)})\n\n        fig = plot(samples)\n        plt.savefig('out/{}.png'\n                    .format(str(i).zfill(3)), bbox_inches='tight')\n        i += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/conditional_gan/cgan_pytorch.py",
    "content": "import torch\nimport torch.nn.functional as nn\nimport torch.autograd as autograd\nimport torch.optim as optim\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 64\nZ_dim = 100\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\ncnt = 0\nlr = 1e-3\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / np.sqrt(in_dim / 2.)\n    return Variable(torch.randn(*size) * xavier_stddev, requires_grad=True)\n\n\n\"\"\" ==================== GENERATOR ======================== \"\"\"\n\nWzh = xavier_init(size=[Z_dim + y_dim, h_dim])\nbzh = Variable(torch.zeros(h_dim), requires_grad=True)\n\nWhx = xavier_init(size=[h_dim, X_dim])\nbhx = Variable(torch.zeros(X_dim), requires_grad=True)\n\n\ndef G(z, c):\n    inputs = torch.cat([z, c], 1)\n    h = nn.relu(inputs @ Wzh + bzh.repeat(inputs.size(0), 1))\n    X = nn.sigmoid(h @ Whx + bhx.repeat(h.size(0), 1))\n    return X\n\n\n\"\"\" ==================== DISCRIMINATOR ======================== \"\"\"\n\nWxh = xavier_init(size=[X_dim + y_dim, h_dim])\nbxh = Variable(torch.zeros(h_dim), requires_grad=True)\n\nWhy = xavier_init(size=[h_dim, 1])\nbhy = Variable(torch.zeros(1), requires_grad=True)\n\n\ndef D(X, c):\n    inputs = torch.cat([X, c], 1)\n    h = nn.relu(inputs @ Wxh + bxh.repeat(inputs.size(0), 1))\n    y = nn.sigmoid(h @ Why + bhy.repeat(h.size(0), 1))\n    return y\n\n\nG_params = [Wzh, bzh, Whx, bhx]\nD_params = [Wxh, bxh, Why, bhy]\nparams = G_params + D_params\n\n\n\"\"\" ===================== TRAINING ======================== \"\"\"\n\n\ndef reset_grad():\n    for p in params:\n        if p.grad is not None:\n            data = p.grad.data\n            p.grad = Variable(data.new().resize_as_(data).zero_())\n\n\nG_solver = optim.Adam(G_params, lr=1e-3)\nD_solver = optim.Adam(D_params, lr=1e-3)\n\nones_label = Variable(torch.ones(mb_size, 1))\nzeros_label = Variable(torch.zeros(mb_size, 1))\n\n\nfor it in range(100000):\n    # Sample data\n    z = Variable(torch.randn(mb_size, Z_dim))\n    X, c = mnist.train.next_batch(mb_size)\n    X = Variable(torch.from_numpy(X))\n    c = Variable(torch.from_numpy(c.astype('float32')))\n\n    # Dicriminator forward-loss-backward-update\n    G_sample = G(z, c)\n    D_real = D(X, c)\n    D_fake = D(G_sample, c)\n\n    D_loss_real = nn.binary_cross_entropy(D_real, ones_label)\n    D_loss_fake = nn.binary_cross_entropy(D_fake, zeros_label)\n    D_loss = D_loss_real + D_loss_fake\n\n    D_loss.backward()\n    D_solver.step()\n\n    # Housekeeping - reset gradient\n    reset_grad()\n\n    # Generator forward-loss-backward-update\n    z = Variable(torch.randn(mb_size, Z_dim))\n    G_sample = G(z, c)\n    D_fake = D(G_sample, c)\n\n    G_loss = nn.binary_cross_entropy(D_fake, ones_label)\n\n    G_loss.backward()\n    G_solver.step()\n\n    # Housekeeping - reset gradient\n    reset_grad()\n\n    # Print and plot every now and then\n    if it % 1000 == 0:\n        print('Iter-{}; D_loss: {}; G_loss: {}'.format(it, D_loss.data.numpy(), G_loss.data.numpy()))\n\n        c = np.zeros(shape=[mb_size, y_dim], dtype='float32')\n        c[:, np.random.randint(0, 10)] = 1.\n        c = Variable(torch.from_numpy(c))\n        samples = G(z, c).data.numpy()[:16]\n\n        fig = plt.figure(figsize=(4, 4))\n        gs = gridspec.GridSpec(4, 4)\n        gs.update(wspace=0.05, hspace=0.05)\n\n        for i, sample in enumerate(samples):\n            ax = plt.subplot(gs[i])\n            plt.axis('off')\n            ax.set_xticklabels([])\n            ax.set_yticklabels([])\n            ax.set_aspect('equal')\n            plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n        if not os.path.exists('out/'):\n            os.makedirs('out/')\n\n        plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight')\n        cnt += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/conditional_gan/cgan_tensorflow.py",
    "content": "import tensorflow as tf\nfrom tensorflow.examples.tutorials.mnist import input_data\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 64\nZ_dim = 100\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)\n    return tf.random_normal(shape=size, stddev=xavier_stddev)\n\n\n\"\"\" Discriminator Net model \"\"\"\nX = tf.placeholder(tf.float32, shape=[None, 784])\ny = tf.placeholder(tf.float32, shape=[None, y_dim])\n\nD_W1 = tf.Variable(xavier_init([X_dim + y_dim, h_dim]))\nD_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\n\nD_W2 = tf.Variable(xavier_init([h_dim, 1]))\nD_b2 = tf.Variable(tf.zeros(shape=[1]))\n\ntheta_D = [D_W1, D_W2, D_b1, D_b2]\n\n\ndef discriminator(x, y):\n    inputs = tf.concat(axis=1, values=[x, y])\n    D_h1 = tf.nn.relu(tf.matmul(inputs, D_W1) + D_b1)\n    D_logit = tf.matmul(D_h1, D_W2) + D_b2\n    D_prob = tf.nn.sigmoid(D_logit)\n\n    return D_prob, D_logit\n\n\n\"\"\" Generator Net model \"\"\"\nZ = tf.placeholder(tf.float32, shape=[None, Z_dim])\n\nG_W1 = tf.Variable(xavier_init([Z_dim + y_dim, h_dim]))\nG_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\n\nG_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nG_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\ntheta_G = [G_W1, G_W2, G_b1, G_b2]\n\n\ndef generator(z, y):\n    inputs = tf.concat(axis=1, values=[z, y])\n    G_h1 = tf.nn.relu(tf.matmul(inputs, G_W1) + G_b1)\n    G_log_prob = tf.matmul(G_h1, G_W2) + G_b2\n    G_prob = tf.nn.sigmoid(G_log_prob)\n\n    return G_prob\n\n\ndef sample_Z(m, n):\n    return np.random.uniform(-1., 1., size=[m, n])\n\n\ndef plot(samples):\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    return fig\n\n\nG_sample = generator(Z, y)\nD_real, D_logit_real = discriminator(X, y)\nD_fake, D_logit_fake = discriminator(G_sample, y)\n\nD_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_real, labels=tf.ones_like(D_logit_real)))\nD_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_fake, labels=tf.zeros_like(D_logit_fake)))\nD_loss = D_loss_real + D_loss_fake\nG_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_fake, labels=tf.ones_like(D_logit_fake)))\n\nD_solver = tf.train.AdamOptimizer().minimize(D_loss, var_list=theta_D)\nG_solver = tf.train.AdamOptimizer().minimize(G_loss, var_list=theta_G)\n\n\nsess = tf.Session()\nsess.run(tf.global_variables_initializer())\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\ni = 0\n\nfor it in range(1000000):\n    if it % 1000 == 0:\n        n_sample = 16\n\n        Z_sample = sample_Z(n_sample, Z_dim)\n        y_sample = np.zeros(shape=[n_sample, y_dim])\n        y_sample[:, 7] = 1\n\n        samples = sess.run(G_sample, feed_dict={Z: Z_sample, y:y_sample})\n\n        fig = plot(samples)\n        plt.savefig('out/{}.png'.format(str(i).zfill(3)), bbox_inches='tight')\n        i += 1\n        plt.close(fig)\n\n    X_mb, y_mb = mnist.train.next_batch(mb_size)\n\n    Z_sample = sample_Z(mb_size, Z_dim)\n    _, D_loss_curr = sess.run([D_solver, D_loss], feed_dict={X: X_mb, Z: Z_sample, y:y_mb})\n    _, G_loss_curr = sess.run([G_solver, G_loss], feed_dict={Z: Z_sample, y:y_mb})\n\n    if it % 1000 == 0:\n        print('Iter: {}'.format(it))\n        print('D loss: {:.4}'. format(D_loss_curr))\n        print('G_loss: {:.4}'.format(G_loss_curr))\n        print()\n"
  },
  {
    "path": "GAN/coupled_gan/cogan_pytorch.py",
    "content": "import torch\nimport torch.nn\nimport torch.nn.functional as nn\nimport torch.autograd as autograd\nimport torch.optim as optim\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\nimport copy\nimport scipy.ndimage.interpolation\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 32\nz_dim = 100\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\ncnt = 0\nlr = 1e-3\n\n\n\"\"\" Shared Generator weights \"\"\"\nG_shared = torch.nn.Sequential(\n    torch.nn.Linear(z_dim, h_dim),\n    torch.nn.ReLU(),\n)\n\n\"\"\" Generator 1 \"\"\"\nG1_ = torch.nn.Sequential(\n    torch.nn.Linear(h_dim, X_dim),\n    torch.nn.Sigmoid()\n)\n\n\"\"\" Generator 2 \"\"\"\nG2_ = torch.nn.Sequential(\n    torch.nn.Linear(h_dim, X_dim),\n    torch.nn.Sigmoid()\n)\n\n\ndef G1(z):\n    h = G_shared(z)\n    X = G1_(h)\n    return X\n\n\ndef G2(z):\n    h = G_shared(z)\n    X = G2_(h)\n    return X\n\n\n\"\"\" Shared Discriminator weights \"\"\"\nD_shared = torch.nn.Sequential(\n    torch.nn.Linear(h_dim, 1),\n    torch.nn.Sigmoid()\n)\n\n\"\"\" Discriminator 1 \"\"\"\nD1_ = torch.nn.Sequential(\n    torch.nn.Linear(X_dim, h_dim),\n    torch.nn.ReLU()\n)\n\n\"\"\" Discriminator 2 \"\"\"\nD2_ = torch.nn.Sequential(\n    torch.nn.Linear(X_dim, h_dim),\n    torch.nn.ReLU()\n)\n\n\ndef D1(X):\n    h = D1_(X)\n    y = D_shared(h)\n    return y\n\n\ndef D2(X):\n    h = D2_(X)\n    y = D_shared(h)\n    return y\n\n\nD_params = (list(D1_.parameters()) + list(D2_.parameters()) +\n            list(D_shared.parameters()))\nG_params = (list(G1_.parameters()) + list(G2_.parameters()) +\n            list(G_shared.parameters()))\nnets = [G_shared, G1_, G2_, D_shared, D1_, D2_]\n\n\ndef reset_grad():\n    for net in nets:\n        net.zero_grad()\n\n\nG_solver = optim.Adam(G_params, lr=lr)\nD_solver = optim.Adam(D_params, lr=lr)\n\nX_train = mnist.train.images\nhalf = int(X_train.shape[0] / 2)\n\n# Real image\nX_train1 = X_train[:half]\n# Rotated image\nX_train2 = X_train[half:].reshape(-1, 28, 28)\nX_train2 = scipy.ndimage.interpolation.rotate(X_train2, 90, axes=(1, 2))\nX_train2 = X_train2.reshape(-1, 28*28)\n\n# Cleanup\ndel X_train\n\n\ndef sample_x(X, size):\n    start_idx = np.random.randint(0, X.shape[0]-size)\n    return Variable(torch.from_numpy(X[start_idx:start_idx+size]))\n\n\nfor it in range(100000):\n    X1 = sample_x(X_train1, mb_size)\n    X2 = sample_x(X_train2, mb_size)\n    z = Variable(torch.randn(mb_size, z_dim))\n\n    # Dicriminator\n    G1_sample = G1(z)\n    D1_real = D1(X1)\n    D1_fake = D1(G1_sample)\n\n    G2_sample = G2(z)\n    D2_real = D2(X2)\n    D2_fake = D2(G2_sample)\n\n    D1_loss = torch.mean(-torch.log(D1_real + 1e-8) -\n                         torch.log(1. - D1_fake + 1e-8))\n    D2_loss = torch.mean(-torch.log(D2_real + 1e-8) -\n                         torch.log(1. - D2_fake + 1e-8))\n    D_loss = D1_loss + D2_loss\n\n    D_loss.backward()\n\n    # Average the gradients\n    for p in D_shared.parameters():\n        p.grad.data = 0.5 * p.grad.data\n\n    D_solver.step()\n    reset_grad()\n\n    # Generator\n    G1_sample = G1(z)\n    D1_fake = D1(G1_sample)\n\n    G2_sample = G2(z)\n    D2_fake = D2(G2_sample)\n\n    G1_loss = torch.mean(-torch.log(D1_fake + 1e-8))\n    G2_loss = torch.mean(-torch.log(D2_fake + 1e-8))\n    G_loss = G1_loss + G2_loss\n\n    G_loss.backward()\n\n    # Average the gradients\n    for p in G_shared.parameters():\n        p.grad.data = 0.5 * p.grad.data\n\n    G_solver.step()\n    reset_grad()\n\n    # Print and plot every now and then\n    if it % 1000 == 0:\n        print('Iter-{}; D1_loss: {:.4}; G1_loss: {:.4}; '\n              'D2_loss: {:.4}; G2_loss: {:.4}'\n              .format(\n                  it, D1_loss.data[0], G1_loss.data[0],\n                  D2_loss.data[0], G2_loss.data[0])\n              )\n\n        z = Variable(torch.randn(8, z_dim))\n        samples1 = G1(z).data.numpy()\n        samples2 = G2(z).data.numpy()\n        samples = np.vstack([samples1, samples2])\n\n        fig = plt.figure(figsize=(4, 4))\n        gs = gridspec.GridSpec(4, 4)\n        gs.update(wspace=0.05, hspace=0.05)\n\n        for i, sample in enumerate(samples):\n            ax = plt.subplot(gs[i])\n            plt.axis('off')\n            ax.set_xticklabels([])\n            ax.set_yticklabels([])\n            ax.set_aspect('equal')\n            plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n        if not os.path.exists('out/'):\n            os.makedirs('out/')\n\n        plt.savefig('out/{}.png'\n                    .format(str(cnt).zfill(3)), bbox_inches='tight')\n        cnt += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/coupled_gan/cogan_tensorflow.py",
    "content": "import tensorflow as tf\nfrom tensorflow.examples.tutorials.mnist import input_data\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nimport scipy.ndimage.interpolation\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\n\nmb_size = 32\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nz_dim = 10\nh_dim = 128\neps = 1e-8\nlr = 1e-3\nd_steps = 3\n\n\ndef plot(samples):\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    return fig\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)\n    return tf.random_normal(shape=size, stddev=xavier_stddev)\n\n\nX1 = tf.placeholder(tf.float32, shape=[None, X_dim])\nX2 = tf.placeholder(tf.float32, shape=[None, X_dim])\nz = tf.placeholder(tf.float32, shape=[None, z_dim])\n\nG_W1 = tf.Variable(xavier_init([z_dim, h_dim]))\nG_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\n\nG1_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nG1_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\nG2_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nG2_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\n\ndef G(z):\n    h = tf.nn.relu(tf.matmul(z, G_W1) + G_b1)\n    G1 = tf.nn.sigmoid(tf.matmul(h, G1_W2) + G1_b2)\n    G2 = tf.nn.sigmoid(tf.matmul(h, G2_W2) + G2_b2)\n    return G1, G2\n\n\nD1_W1 = tf.Variable(xavier_init([X_dim, h_dim]))\nD1_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\n\nD2_W1 = tf.Variable(xavier_init([X_dim, h_dim]))\nD2_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\n\nD_W2 = tf.Variable(xavier_init([h_dim, 1]))\nD_b2 = tf.Variable(tf.zeros(shape=[1]))\n\n\ndef D(X1, X2):\n    h1 = tf.nn.relu(tf.matmul(X1, D1_W1) + D1_b1)\n    h2 = tf.nn.relu(tf.matmul(X2, D2_W1) + D2_b1)\n    D1_out = tf.nn.sigmoid(tf.matmul(h1, D_W2) + D_b2)\n    D2_out = tf.nn.sigmoid(tf.matmul(h2, D_W2) + D_b2)\n    return D1_out, D2_out\n\n\ntheta_G = [G1_W2, G2_W2, G1_b2, G2_b2]\ntheta_G_shared = [G_W1, G_b1]\n\ntheta_D = [D1_W1, D2_W1, D1_b1, D2_b1]\ntheta_D_shared = [D_W2, D_b2]\n\n# Train D\nG1_sample, G2_sample = G(z)\nD1_real, D2_real = D(X1, X2)\nD1_fake, D2_fake = D(G1_sample, G2_sample)\n\nD1_loss = -tf.reduce_mean(tf.log(D1_real + eps) + tf.log(1. - D1_fake + eps))\nD2_loss = -tf.reduce_mean(tf.log(D2_real + eps) + tf.log(1. - D2_fake + eps))\nD_loss = D1_loss + D2_loss\n\n# Train G\nG1_loss = -tf.reduce_mean(tf.log(D1_fake + eps))\nG2_loss = -tf.reduce_mean(tf.log(D2_fake + eps))\nG_loss = G1_loss + G2_loss\n\n# D optimizer\nD_opt = tf.train.AdamOptimizer(learning_rate=lr)\n# Compute the gradients for a list of variables.\nD_gv = D_opt.compute_gradients(D_loss, theta_D)\nD_shared_gv = D_opt.compute_gradients(D_loss, theta_D_shared)\n# Average by halfing the shared gradients\nD_shared_gv = [(0.5 * x[0], x[1]) for x in D_shared_gv]\n# Update\nD_solver = tf.group(\n    D_opt.apply_gradients(D_gv), D_opt.apply_gradients(D_shared_gv)\n)\n\n# G optimizer\nG_opt = tf.train.AdamOptimizer(learning_rate=lr)\n# Compute the gradients for a list of variables.\nG_gv = G_opt.compute_gradients(G_loss, theta_G)\nG_shared_gv = G_opt.compute_gradients(G_loss, theta_G_shared)\n# Average by halfing the shared gradients\nG_shared_gv = [(0.5 * x[0], x[1]) for x in G_shared_gv]\n# Update\nG_solver = tf.group(\n    G_opt.apply_gradients(G_gv), G_opt.apply_gradients(G_shared_gv)\n)\n\nsess = tf.Session()\nsess.run(tf.global_variables_initializer())\n\nX_train = mnist.train.images\nhalf = int(X_train.shape[0] / 2)\n\n# Real image\nX_train1 = X_train[:half]\n# Rotated image\nX_train2 = X_train[half:].reshape(-1, 28, 28)\nX_train2 = scipy.ndimage.interpolation.rotate(X_train2, 90, axes=(1, 2))\nX_train2 = X_train2.reshape(-1, 28*28)\n\n# Cleanup\ndel X_train\n\n\ndef sample_X(X, size):\n    start_idx = np.random.randint(0, X.shape[0]-size)\n    return X[start_idx:start_idx+size]\n\n\ndef sample_z(m, n):\n    return np.random.uniform(-1., 1., size=[m, n])\n\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\ni = 0\n\nfor it in range(1000000):\n    X1_mb, X2_mb = sample_X(X_train1, mb_size), sample_X(X_train2, mb_size)\n    z_mb = sample_z(mb_size, z_dim)\n\n    _, D_loss_curr = sess.run(\n        [D_solver, D_loss],\n        feed_dict={X1: X1_mb, X2: X2_mb, z: z_mb}\n    )\n\n    _, G_loss_curr = sess.run(\n        [G_solver, G_loss], feed_dict={z: z_mb}\n    )\n\n    if it % 1000 == 0:\n        sample1, sample2 = sess.run(\n            [G1_sample, G2_sample], feed_dict={z: sample_z(8, z_dim)}\n        )\n\n        samples = np.vstack([sample1, sample2])\n\n        print('Iter: {}; D_loss: {:.4}; G_loss: {:.4}'\n              .format(it, D_loss_curr, G_loss_curr))\n\n        fig = plot(samples)\n        plt.savefig('out/{}.png'\n                    .format(str(i).zfill(3)), bbox_inches='tight')\n        i += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/disco_gan/discogan_pytorch.py",
    "content": "import torch\nimport torch.nn\nimport torch.nn.functional as nn\nimport torch.autograd as autograd\nimport torch.optim as optim\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\nimport scipy.ndimage.interpolation\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 32\nz_dim = 10\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\ncnt = 0\nlr = 1e-3\n\n\ndef log(x):\n    return torch.log(x + 1e-8)\n\n\ndef plot(samples):\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    return fig\n\n\nG_AB = torch.nn.Sequential(\n    torch.nn.Linear(X_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, X_dim),\n    torch.nn.Sigmoid()\n)\n\nG_BA = torch.nn.Sequential(\n    torch.nn.Linear(X_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, X_dim),\n    torch.nn.Sigmoid()\n)\n\nD_A = torch.nn.Sequential(\n    torch.nn.Linear(X_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, 1),\n    torch.nn.Sigmoid()\n)\n\nD_B = torch.nn.Sequential(\n    torch.nn.Linear(X_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, 1),\n    torch.nn.Sigmoid()\n)\n\nnets = [G_AB, G_BA, D_A, D_B]\nG_params = list(G_AB.parameters()) + list(G_BA.parameters())\nD_params = list(D_A.parameters()) + list(D_B.parameters())\n\n\ndef reset_grad():\n    for net in nets:\n        net.zero_grad()\n\n\nG_solver = optim.Adam(G_params, lr=lr)\nD_solver = optim.Adam(D_params, lr=lr)\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\n# Gather training data: domain1 <- real MNIST img, domain2 <- rotated MNIST img\nX_train = mnist.train.images\nhalf = int(X_train.shape[0] / 2)\n# Real image\nX_train1 = X_train[:half]\n# Rotated image\nX_train2 = X_train[half:].reshape(-1, 28, 28)\nX_train2 = scipy.ndimage.interpolation.rotate(X_train2, 90, axes=(1, 2))\nX_train2 = X_train2.reshape(-1, 28*28)\n# Cleanup\ndel X_train\n\n\ndef sample_x(X, size):\n    start_idx = np.random.randint(0, X.shape[0]-size)\n    return Variable(torch.from_numpy(X[start_idx:start_idx+size]))\n\n\n# Training\nfor it in range(1000000):\n    # Sample data from both domains\n    X_A = sample_x(X_train1, mb_size)\n    X_B = sample_x(X_train2, mb_size)\n\n    # Discriminator A\n    X_BA = G_BA(X_B)\n    D_A_real = D_A(X_A)\n    D_A_fake = D_A(X_BA)\n\n    L_D_A = -torch.mean(log(D_A_real) + log(1 - D_A_fake))\n\n    # Discriminator B\n    X_AB = G_AB(X_A)\n    D_B_real = D_B(X_B)\n    D_B_fake = D_B(X_AB)\n\n    L_D_B = -torch.mean(log(D_B_real) + log(1 - D_B_fake))\n\n    # Total discriminator loss\n    D_loss = L_D_A + L_D_B\n\n    D_loss.backward()\n    D_solver.step()\n    reset_grad()\n\n    # Generator AB\n    X_AB = G_AB(X_A)\n    D_B_fake = D_B(X_AB)\n    X_ABA = G_BA(X_AB)\n\n    L_adv_B = -torch.mean(log(D_B_fake))\n    L_recon_A = torch.mean(torch.sum((X_A - X_ABA)**2, 1))\n    L_G_AB = L_adv_B + L_recon_A\n\n    # Generator BA\n    X_BA = G_BA(X_B)\n    D_A_fake = D_A(X_BA)\n    X_BAB = G_AB(X_BA)\n\n    L_adv_A = -torch.mean(log(D_A_fake))\n    L_recon_B = torch.mean(torch.sum((X_B - X_BAB)**2, 1))\n    L_G_BA = L_adv_A + L_recon_B\n\n    # Total generator loss\n    G_loss = L_G_AB + L_G_BA\n\n    G_loss.backward()\n    G_solver.step()\n    reset_grad()\n\n    # Print and plot every now and then\n    if it % 1000 == 0:\n        print('Iter-{}; D_loss: {:.4}; G_loss: {:.4}'\n              .format(it, D_loss.data[0], G_loss.data[0]))\n\n        input_A = sample_x(X_train1, size=4)\n        input_B = sample_x(X_train2, size=4)\n\n        samples_A = G_BA(input_B).data.numpy()\n        samples_B = G_AB(input_A).data.numpy()\n\n        input_A = input_A.data.numpy()\n        input_B = input_B.data.numpy()\n\n        # The resulting image sample would be in 4 rows:\n        # row 1: real data from domain A, row 2 is its domain B translation\n        # row 3: real data from domain B, row 4 is its domain A translation\n        samples = np.vstack([input_A, samples_B, input_B, samples_A])\n\n        fig = plot(samples)\n        plt.savefig('out/{}.png'\n                    .format(str(cnt).zfill(3)), bbox_inches='tight')\n        cnt += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/disco_gan/discogan_tensorflow.py",
    "content": "import tensorflow as tf\nfrom tensorflow.examples.tutorials.mnist import input_data\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nimport scipy.ndimage.interpolation\n\n\nmb_size = 32\nX_dim = 784\nz_dim = 64\nh_dim = 128\nlr = 1e-3\nd_steps = 3\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\n\n\ndef plot(samples):\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    return fig\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)\n    return tf.random_normal(shape=size, stddev=xavier_stddev)\n\n\ndef log(x):\n    return tf.log(x + 1e-8)\n\n\nX_A = tf.placeholder(tf.float32, shape=[None, X_dim])\nX_B = tf.placeholder(tf.float32, shape=[None, X_dim])\n\nD_A_W1 = tf.Variable(xavier_init([X_dim, h_dim]))\nD_A_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nD_A_W2 = tf.Variable(xavier_init([h_dim, 1]))\nD_A_b2 = tf.Variable(tf.zeros(shape=[1]))\n\nD_B_W1 = tf.Variable(xavier_init([X_dim, h_dim]))\nD_B_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nD_B_W2 = tf.Variable(xavier_init([h_dim, 1]))\nD_B_b2 = tf.Variable(tf.zeros(shape=[1]))\n\nG_AB_W1 = tf.Variable(xavier_init([X_dim, h_dim]))\nG_AB_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nG_AB_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nG_AB_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\nG_BA_W1 = tf.Variable(xavier_init([X_dim, h_dim]))\nG_BA_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nG_BA_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nG_BA_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\ntheta_D = [D_A_W1, D_A_W2, D_A_b1, D_A_b2,\n           D_B_W1, D_B_W2, D_B_b1, D_B_b2]\ntheta_G = [G_AB_W1, G_AB_W2, G_AB_b1, G_AB_b2,\n           G_BA_W1, G_BA_W2, G_BA_b1, G_BA_b2]\n\n\ndef D_A(X):\n    h = tf.nn.relu(tf.matmul(X, D_A_W1) + D_A_b1)\n    return tf.nn.sigmoid(tf.matmul(h, D_A_W2) + D_A_b2)\n\n\ndef D_B(X):\n    h = tf.nn.relu(tf.matmul(X, D_B_W1) + D_B_b1)\n    return tf.nn.sigmoid(tf.matmul(h, D_B_W2) + D_B_b2)\n\n\ndef G_AB(X):\n    h = tf.nn.relu(tf.matmul(X, G_AB_W1) + G_AB_b1)\n    return tf.nn.sigmoid(tf.matmul(h, G_AB_W2) + G_AB_b2)\n\n\ndef G_BA(X):\n    h = tf.nn.relu(tf.matmul(X, G_BA_W1) + G_BA_b1)\n    return tf.nn.sigmoid(tf.matmul(h, G_BA_W2) + G_BA_b2)\n\n\n# Discriminator A\nX_BA = G_BA(X_B)\nD_A_real = D_A(X_A)\nD_A_fake = D_A(X_BA)\n\n# Discriminator B\nX_AB = G_AB(X_A)\nD_B_real = D_B(X_B)\nD_B_fake = D_B(X_AB)\n\n# Generator AB\nX_ABA = G_BA(X_AB)\n\n# Generator BA\nX_BAB = G_AB(X_BA)\n\n# Discriminator loss\nL_D_A = -tf.reduce_mean(log(D_A_real) + log(1 - D_A_fake))\nL_D_B = -tf.reduce_mean(log(D_B_real) + log(1 - D_B_fake))\n\nD_loss = L_D_A + L_D_B\n\n# Generator loss\nL_adv_B = -tf.reduce_mean(log(D_B_fake))\nL_recon_A = tf.reduce_mean(tf.reduce_sum((X_A - X_ABA)**2, 1))\nL_G_AB = L_adv_B + L_recon_A\n\nL_adv_A = -tf.reduce_mean(log(D_A_fake))\nL_recon_B = tf.reduce_mean(tf.reduce_sum((X_B - X_BAB)**2, 1))\nL_G_BA = L_adv_A + L_recon_B\n\nG_loss = L_G_AB + L_G_BA\n\n# Solvers\nsolver = tf.train.AdamOptimizer(learning_rate=lr)\nD_solver = solver.minimize(D_loss, var_list=theta_D)\nG_solver = solver.minimize(G_loss, var_list=theta_G)\n\nsess = tf.Session()\nsess.run(tf.global_variables_initializer())\n\n\n# Gather training data from 2 domains\nX_train = mnist.train.images\nhalf = int(X_train.shape[0] / 2)\n# Real image\nX_train1 = X_train[:half]\n# Rotated image\nX_train2 = X_train[half:].reshape(-1, 28, 28)\nX_train2 = scipy.ndimage.interpolation.rotate(X_train2, 90, axes=(1, 2))\nX_train2 = X_train2.reshape(-1, 28*28)\n# Cleanup\ndel X_train\n\n\ndef sample_X(X, size):\n    start_idx = np.random.randint(0, X.shape[0]-size)\n    return X[start_idx:start_idx+size]\n\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\ni = 0\n\nfor it in range(1000000):\n    # Sample data from both domains\n    X_A_mb = sample_X(X_train1, mb_size)\n    X_B_mb = sample_X(X_train2, mb_size)\n\n    _, D_loss_curr = sess.run(\n        [D_solver, D_loss], feed_dict={X_A: X_A_mb, X_B: X_B_mb}\n    )\n\n    _, G_loss_curr = sess.run(\n        [G_solver, G_loss], feed_dict={X_A: X_A_mb, X_B: X_B_mb}\n    )\n\n    if it % 1000 == 0:\n        print('Iter: {}; D_loss: {:.4}; G_loss: {:.4}'\n              .format(it, D_loss_curr, G_loss_curr))\n\n        input_A = sample_X(X_train1, size=4)\n        input_B = sample_X(X_train2, size=4)\n\n        samples_A = sess.run(X_BA, feed_dict={X_B: input_B})\n        samples_B = sess.run(X_AB, feed_dict={X_A: input_A})\n\n        # The resulting image sample would be in 4 rows:\n        # row 1: real data from domain A, row 2 is its domain B translation\n        # row 3: real data from domain B, row 4 is its domain A translation\n        samples = np.vstack([input_A, samples_B, input_B, samples_A])\n\n        fig = plot(samples)\n        plt.savefig('out/{}.png'\n                    .format(str(i).zfill(3)), bbox_inches='tight')\n        i += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/dual_gan/dualgan_pytorch.py",
    "content": "import torch\nimport torch.nn\nimport torch.nn.functional as nn\nimport torch.autograd as autograd\nimport torch.optim as optim\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\nfrom itertools import chain\nimport scipy.ndimage.interpolation\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 32\nz_dim = 10\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\ncnt = 0\nlr = 1e-4\nn_critics = 3\nlam1, lam2 = 100, 100\n\n\ndef log(x):\n    return torch.log(x + 1e-8)\n\n\nG1 = torch.nn.Sequential(\n    torch.nn.Linear(X_dim + z_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, X_dim),\n    torch.nn.Sigmoid()\n)\n\nG2 = torch.nn.Sequential(\n    torch.nn.Linear(X_dim + z_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, X_dim),\n    torch.nn.Sigmoid()\n)\n\nD1 = torch.nn.Sequential(\n    torch.nn.Linear(X_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, 1)\n)\n\nD2 = torch.nn.Sequential(\n    torch.nn.Linear(X_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, 1)\n)\n\n\ndef reset_grad():\n    G1.zero_grad()\n    G2.zero_grad()\n    D1.zero_grad()\n    D2.zero_grad()\n\n\nG_solver = optim.RMSprop(chain(G1.parameters(), G2.parameters()), lr=lr)\nD1_solver = optim.RMSprop(D1.parameters(), lr=lr)\nD2_solver = optim.RMSprop(D2.parameters(), lr=lr)\n\nX_train = mnist.train.images\nhalf = int(X_train.shape[0] / 2)\n\n# Real image\nX_train1 = X_train[:half]\n# Rotated image\nX_train2 = X_train[half:].reshape(-1, 28, 28)\nX_train2 = scipy.ndimage.interpolation.rotate(X_train2, 90, axes=(1, 2))\nX_train2 = X_train2.reshape(-1, 28*28)\n\n# Cleanup\ndel X_train\n\n\ndef sample_x(X, size):\n    start_idx = np.random.randint(0, X.shape[0]-size)\n    return Variable(torch.from_numpy(X[start_idx:start_idx+size]))\n\n\nfor it in range(1000000):\n    for _ in range(n_critics):\n        # Sample data\n        z1 = Variable(torch.randn(mb_size, z_dim))\n        z2 = Variable(torch.randn(mb_size, z_dim))\n        X1 = sample_x(X_train1, mb_size)\n        X2 = sample_x(X_train2, mb_size)\n\n        # D1\n        X2_sample = G1(torch.cat([X1, z1], 1))  # G1: X1 -> X2\n        D1_real = D1(X2)\n        D1_fake = D1(X2_sample)\n\n        D1_loss = -(torch.mean(D1_real) - torch.mean(D1_fake))\n\n        D1_loss.backward(retain_graph=True)\n        D1_solver.step()\n\n        # Weight clipping\n        for p in D1.parameters():\n            p.data.clamp_(-0.01, 0.01)\n\n        reset_grad()\n\n        # D2\n        X1_sample = G2(torch.cat([X2, z2], 1))  # G2: X2 -> X1\n        D2_real = D2(X1)\n        D2_fake = D2(X1_sample)\n\n        D2_loss = -(torch.mean(D2_real) - torch.mean(D2_fake))\n\n        D2_loss.backward()\n        D2_solver.step()\n\n        # Weight clipping\n        for p in D2.parameters():\n            p.data.clamp_(-0.01, 0.01)\n\n        reset_grad()\n\n    # Generator\n    z1 = Variable(torch.randn(mb_size, z_dim))\n    z2 = Variable(torch.randn(mb_size, z_dim))\n    X1 = sample_x(X_train1, mb_size)\n    X2 = sample_x(X_train2, mb_size)\n\n    X1_sample = G2(torch.cat([X2, z2], 1))\n    X2_sample = G1(torch.cat([X1, z1], 1))\n\n    X1_recon = G2(torch.cat([X2_sample, z2], 1))\n    X2_recon = G1(torch.cat([X1_sample, z1], 1))\n\n    D1_fake = D1(X1_sample)\n    D2_fake = D2(X2_sample)\n\n    G_loss = -torch.mean(D1_fake) - torch.mean(D2_fake)\n    reg1 = lam1 * torch.mean(torch.sum(torch.abs(X1_recon - X1), 1))\n    reg2 = lam2 * torch.mean(torch.sum(torch.abs(X2_recon - X2), 1))\n\n    G_loss += reg1 + reg2\n\n    G_loss.backward()\n    G_solver.step()\n    reset_grad()\n\n    # Print and plot every now and then\n    if it % 1000 == 0:\n        print('Iter-{}; D_loss: {:.4}; G_loss: {:.4}'\n              .format(it, D1_loss.data[0] + D2_loss.data[0], G_loss.data[0]))\n\n        real1 = X1.data.numpy()[:4]\n        real2 = X2.data.numpy()[:4]\n        samples1 = X1_sample.data.numpy()[:4]\n        samples2 = X2_sample.data.numpy()[:4]\n        samples = np.vstack([real2, samples1, real1, samples2])\n\n        fig = plt.figure(figsize=(4, 4))\n        gs = gridspec.GridSpec(4, 4)\n        gs.update(wspace=0.05, hspace=0.05)\n\n        for i, sample in enumerate(samples):\n            ax = plt.subplot(gs[i])\n            plt.axis('off')\n            ax.set_xticklabels([])\n            ax.set_yticklabels([])\n            ax.set_aspect('equal')\n            plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n        if not os.path.exists('out/'):\n            os.makedirs('out/')\n\n        plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight')\n        cnt += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/dual_gan/dualgan_tensorflow.py",
    "content": "import tensorflow as tf\nfrom tensorflow.examples.tutorials.mnist import input_data\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nimport scipy.ndimage.interpolation\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\n\nmb_size = 32\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nz_dim = 10\nh_dim = 128\neps = 1e-8\nlr = 1e-3\nd_steps = 3\nlam1, lam2 = 1000, 1000\n\n\ndef plot(samples):\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    return fig\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)\n    return tf.random_normal(shape=size, stddev=xavier_stddev)\n\n\nX1 = tf.placeholder(tf.float32, shape=[None, X_dim])\nX2 = tf.placeholder(tf.float32, shape=[None, X_dim])\nz = tf.placeholder(tf.float32, shape=[None, z_dim])\n\nG1_W1 = tf.Variable(xavier_init([X_dim + z_dim, h_dim]))\nG1_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nG1_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nG1_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\nG2_W1 = tf.Variable(xavier_init([X_dim + z_dim, h_dim]))\nG2_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nG2_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nG2_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\n\ndef G1(X1, z):\n    inputs = tf.concat([X1, z], 1)\n    h = tf.nn.relu(tf.matmul(inputs, G1_W1) + G1_b1)\n    return tf.nn.sigmoid(tf.matmul(h, G1_W2) + G1_b2)\n\n\ndef G2(X2, z):\n    inputs = tf.concat([X2, z], 1)\n    h = tf.nn.relu(tf.matmul(inputs, G2_W1) + G2_b1)\n    return tf.nn.sigmoid(tf.matmul(h, G2_W2) + G2_b2)\n\n\nD1_W1 = tf.Variable(xavier_init([X_dim, h_dim]))\nD1_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nD1_W2 = tf.Variable(xavier_init([h_dim, 1]))\nD1_b2 = tf.Variable(tf.zeros(shape=[1]))\n\nD2_W1 = tf.Variable(xavier_init([X_dim, h_dim]))\nD2_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nD2_W2 = tf.Variable(xavier_init([h_dim, 1]))\nD2_b2 = tf.Variable(tf.zeros(shape=[1]))\n\n\ndef D1(X):\n    h = tf.nn.relu(tf.matmul(X, D1_W1) + D1_b1)\n    return tf.matmul(h, D1_W2) + D1_b2\n\n\ndef D2(X):\n    h = tf.nn.relu(tf.matmul(X, D1_W1) + D1_b1)\n    return tf.matmul(h, D2_W2) + D2_b2\n\n\ntheta_G1 = [G1_W1, G1_W2, G1_b2, G1_b2]\ntheta_G2 = [G2_W1, G2_b1, G2_W2, G2_b2]\ntheta_G = theta_G1 + theta_G2\n\ntheta_D1 = [D1_W1, D1_W2, D1_b1, D1_b2]\ntheta_D2 = [D2_W1, D2_b1, D2_W2, D2_b2]\n\n# D\nX1_sample = G2(X2, z)\nX2_sample = G1(X1, z)\n\nD1_real = D1(X2)\nD1_fake = D1(X2_sample)\n\nD2_real = D2(X1)\nD2_fake = D2(X1_sample)\n\nD1_G = D1(X1_sample)\nD2_G = D2(X2_sample)\n\nX1_recon = G2(X2_sample, z)\nX2_recon = G1(X1_sample, z)\nrecon1 = tf.reduce_mean(tf.reduce_sum(tf.abs(X1 - X1_recon), 1))\nrecon2 = tf.reduce_mean(tf.reduce_sum(tf.abs(X2 - X2_recon), 1))\n\nD1_loss = tf.reduce_mean(D1_fake) - tf.reduce_mean(D1_real)\nD2_loss = tf.reduce_mean(D2_fake) - tf.reduce_mean(D2_real)\nG_loss = -tf.reduce_mean(D1_G + D2_G) + lam1*recon1 + lam2*recon2\n\nD1_solver = (tf.train.RMSPropOptimizer(learning_rate=1e-4)\n             .minimize(D1_loss, var_list=theta_D1))\nD2_solver = (tf.train.RMSPropOptimizer(learning_rate=1e-4)\n             .minimize(D2_loss, var_list=theta_D2))\nG_solver = (tf.train.RMSPropOptimizer(learning_rate=1e-4)\n            .minimize(G_loss, var_list=theta_G))\n\nclip_D = [p.assign(tf.clip_by_value(p, -0.01, 0.01)) for p in theta_D1 + theta_D2]\n\n\nsess = tf.Session()\nsess.run(tf.global_variables_initializer())\n\nX_train = mnist.train.images\nhalf = int(X_train.shape[0] / 2)\n\n# Real image\nX_train1 = X_train[:half]\n# Rotated image\nX_train2 = X_train[half:].reshape(-1, 28, 28)\nX_train2 = scipy.ndimage.interpolation.rotate(X_train2, 90, axes=(1, 2))\nX_train2 = X_train2.reshape(-1, 28*28)\n\n# Cleanup\ndel X_train\n\n\ndef sample_X(X, size):\n    start_idx = np.random.randint(0, X.shape[0]-size)\n    return X[start_idx:start_idx+size]\n\n\ndef sample_z(m, n):\n    return np.random.uniform(-1., 1., size=[m, n])\n\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\ni = 0\n\nfor it in range(1000000):\n    for _ in range(d_steps):\n        X1_mb, X2_mb = sample_X(X_train1, mb_size), sample_X(X_train2, mb_size)\n        z_mb = sample_z(mb_size, z_dim)\n\n        _, _, D1_loss_curr, D2_loss_curr, _ = sess.run(\n            [D1_solver, D2_solver, D1_loss, D2_loss, clip_D],\n            feed_dict={X1: X1_mb, X2: X2_mb, z: z_mb}\n        )\n\n    _, G_loss_curr = sess.run(\n        [G_solver, G_loss], feed_dict={X1: X1_mb, X2: X2_mb, z: z_mb}\n    )\n\n    if it % 1000 == 0:\n        sample1, sample2 = sess.run(\n            [X1_sample, X2_sample],\n            feed_dict={X1: X1_mb[:4], X2: X2_mb[:4], z: sample_z(4, z_dim)}\n        )\n\n        samples = np.vstack([X1_mb[:4], sample1, X2_mb[:4], sample2])\n\n        print('Iter: {}; D_loss: {:.4}; G_loss: {:.4}'\n              .format(it, D1_loss_curr + D2_loss_curr, G_loss_curr))\n\n        fig = plot(samples)\n        plt.savefig('out/{}.png'\n                    .format(str(i).zfill(3)), bbox_inches='tight')\n        i += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/ebgan/ebgan_pytorch.py",
    "content": "import torch\nimport torch.nn\nimport torch.nn.functional as nn\nimport torch.autograd as autograd\nimport torch.optim as optim\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 32\nz_dim = 10\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\ncnt = 0\nd_step = 3\nlr = 1e-3\nm = 5\n\n\nG = torch.nn.Sequential(\n    torch.nn.Linear(z_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, X_dim),\n    torch.nn.Sigmoid()\n)\n\n# D is an autoencoder\nD_ = torch.nn.Sequential(\n    torch.nn.Linear(X_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, X_dim),\n)\n\n\n# Energy is the MSE of autoencoder\ndef D(X):\n    X_recon = D_(X)\n    return torch.mean(torch.sum((X - X_recon)**2, 1))\n\n\ndef reset_grad():\n    G.zero_grad()\n    D_.zero_grad()\n\n\nG_solver = optim.Adam(G.parameters(), lr=lr)\nD_solver = optim.Adam(D_.parameters(), lr=lr)\n\n\nfor it in range(1000000):\n    # Sample data\n    z = Variable(torch.randn(mb_size, z_dim))\n    X, _ = mnist.train.next_batch(mb_size)\n    X = Variable(torch.from_numpy(X))\n\n    # Dicriminator\n    G_sample = G(z)\n    D_real = D(X)\n    D_fake = D(G_sample)\n\n    # EBGAN D loss. D_real and D_fake is energy, i.e. a number\n    D_loss = D_real + nn.relu(m - D_fake)\n\n    # Reuse D_fake for generator loss\n    D_loss.backward()\n    D_solver.step()\n    reset_grad()\n\n    # Generator\n    G_sample = G(z)\n    D_fake = D(G_sample)\n\n    G_loss = D_fake\n\n    G_loss.backward()\n    G_solver.step()\n    reset_grad()\n\n    # Print and plot every now and then\n    if it % 1000 == 0:\n        print('Iter-{}; D_loss: {:.4}; G_loss: {:.4}'\n              .format(it, D_loss.data[0], G_loss.data[0]))\n\n        samples = G(z).data.numpy()[:16]\n\n        fig = plt.figure(figsize=(4, 4))\n        gs = gridspec.GridSpec(4, 4)\n        gs.update(wspace=0.05, hspace=0.05)\n\n        for i, sample in enumerate(samples):\n            ax = plt.subplot(gs[i])\n            plt.axis('off')\n            ax.set_xticklabels([])\n            ax.set_yticklabels([])\n            ax.set_aspect('equal')\n            plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n        if not os.path.exists('out/'):\n            os.makedirs('out/')\n\n        plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight')\n        cnt += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/ebgan/ebgan_tensorflow.py",
    "content": "import tensorflow as tf\nfrom tensorflow.examples.tutorials.mnist import input_data\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\n\n\nmb_size = 32\nX_dim = 784\nz_dim = 64\nh_dim = 128\nlr = 1e-3\nm = 5\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\n\n\ndef plot(samples):\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    return fig\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)\n    return tf.random_normal(shape=size, stddev=xavier_stddev)\n\n\nX = tf.placeholder(tf.float32, shape=[None, X_dim])\nz = tf.placeholder(tf.float32, shape=[None, z_dim])\n\nD_W1 = tf.Variable(xavier_init([X_dim, h_dim]))\nD_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nD_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nD_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\nG_W1 = tf.Variable(xavier_init([z_dim, h_dim]))\nG_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nG_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nG_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\ntheta_G = [G_W1, G_W2, G_b1, G_b2]\ntheta_D = [D_W1, D_W2, D_b1, D_b2]\n\n\ndef sample_z(m, n):\n    return np.random.uniform(-1., 1., size=[m, n])\n\n\ndef generator(z):\n    G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1)\n    G_log_prob = tf.matmul(G_h1, G_W2) + G_b2\n    G_prob = tf.nn.sigmoid(G_log_prob)\n    return G_prob\n\n\ndef discriminator(X):\n    D_h1 = tf.nn.relu(tf.matmul(X, D_W1) + D_b1)\n    X_recon = tf.matmul(D_h1, D_W2) + D_b2\n    mse = tf.reduce_mean(tf.reduce_sum((X - X_recon)**2, 1))\n    return mse\n\n\nG_sample = generator(z)\n\nD_real = discriminator(X)\nD_fake = discriminator(G_sample)\n\nD_loss = D_real + tf.maximum(0., m - D_fake)\nG_loss = D_fake\n\nD_solver = (tf.train.AdamOptimizer(learning_rate=lr)\n            .minimize(D_loss, var_list=theta_D))\nG_solver = (tf.train.AdamOptimizer(learning_rate=lr)\n            .minimize(G_loss, var_list=theta_G))\n\nsess = tf.Session()\nsess.run(tf.global_variables_initializer())\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\ni = 0\n\nfor it in range(1000000):\n    X_mb, _ = mnist.train.next_batch(mb_size)\n    z_mb = sample_z(mb_size, z_dim)\n\n    _, D_loss_curr = sess.run([D_solver, D_loss], feed_dict={X: X_mb, z: z_mb})\n\n    _, G_loss_curr = sess.run(\n        [G_solver, G_loss], feed_dict={X: X_mb, z: sample_z(mb_size, z_dim)}\n    )\n\n    if it % 1000 == 0:\n        print('Iter: {}; D_loss: {:.4}; G_loss: {:.4}'\n              .format(it, D_loss_curr, G_loss_curr))\n\n        samples = sess.run(G_sample, feed_dict={z: sample_z(16, z_dim)})\n\n        fig = plot(samples)\n        plt.savefig('out/{}.png'\n                    .format(str(i).zfill(3)), bbox_inches='tight')\n        i += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/f_gan/f_gan_pytorch.py",
    "content": "import torch\nimport torch.nn\nimport torch.nn.functional as nn\nimport torch.autograd as autograd\nimport torch.optim as optim\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 32\nz_dim = 10\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\ncnt = 0\nlr = 1e-3\n\n\ndef log(x):\n    return torch.log(x + 1e-8)\n\n\nG = torch.nn.Sequential(\n    torch.nn.Linear(z_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, X_dim),\n    torch.nn.Sigmoid()\n)\n\n\nD = torch.nn.Sequential(\n    torch.nn.Linear(X_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, 1),\n)\n\n\ndef reset_grad():\n    G.zero_grad()\n    D.zero_grad()\n\n\nG_solver = optim.Adam(G.parameters(), lr=lr)\nD_solver = optim.Adam(D.parameters(), lr=lr)\n\n\nfor it in range(1000000):\n    # Sample data\n    z = Variable(torch.randn(mb_size, z_dim))\n    X, _ = mnist.train.next_batch(mb_size)\n    X = Variable(torch.from_numpy(X))\n\n    # Dicriminator\n    G_sample = G(z)\n    D_real = D(X)\n    D_fake = D(G_sample)\n\n    # Uncomment D_loss and its respective G_loss of your choice\n    # ---------------------------------------------------------\n\n    \"\"\" Total Variation \"\"\"\n    # D_loss = -(torch.mean(0.5 * torch.tanh(D_real)) -\n    #            torch.mean(0.5 * torch.tanh(D_fake)))\n    \"\"\" Forward KL \"\"\"\n    # D_loss = -(torch.mean(D_real) - torch.mean(torch.exp(D_fake - 1)))\n    \"\"\" Reverse KL \"\"\"\n    D_loss = -(torch.mean(-torch.exp(D_real)) - torch.mean(-1 - D_fake))\n    \"\"\" Pearson Chi-squared \"\"\"\n    # D_loss = -(torch.mean(D_real) - torch.mean(0.25*D_fake**2 + D_fake))\n    \"\"\" Squared Hellinger \"\"\"\n    # D_loss = -(torch.mean(1 - torch.exp(D_real)) -\n    #            torch.mean((1 - torch.exp(D_fake)) / (torch.exp(D_fake))))\n\n    D_loss.backward()\n    D_solver.step()\n    reset_grad()\n\n    # Generator\n    G_sample = G(z)\n    D_fake = D(G_sample)\n\n    \"\"\" Total Variation \"\"\"\n    # G_loss = -torch.mean(0.5 * torch.tanh(D_fake))\n    \"\"\" Forward KL \"\"\"\n    # G_loss = -torch.mean(torch.exp(D_fake - 1))\n    \"\"\" Reverse KL \"\"\"\n    G_loss = -torch.mean(-1 - D_fake)\n    \"\"\" Pearson Chi-squared \"\"\"\n    # G_loss = -torch.mean(0.25*D_fake**2 + D_fake)\n    \"\"\" Squared Hellinger \"\"\"\n    # G_loss = -torch.mean((1 - torch.exp(D_fake)) / (torch.exp(D_fake)))\n\n    G_loss.backward()\n    G_solver.step()\n    reset_grad()\n\n    # Print and plot every now and then\n    if it % 1000 == 0:\n        print('Iter-{}; D_loss: {:.4}; G_loss: {:.4}'\n              .format(it, D_loss.data[0], G_loss.data[0]))\n\n        samples = G(z).data.numpy()[:16]\n\n        fig = plt.figure(figsize=(4, 4))\n        gs = gridspec.GridSpec(4, 4)\n        gs.update(wspace=0.05, hspace=0.05)\n\n        for i, sample in enumerate(samples):\n            ax = plt.subplot(gs[i])\n            plt.axis('off')\n            ax.set_xticklabels([])\n            ax.set_yticklabels([])\n            ax.set_aspect('equal')\n            plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n        if not os.path.exists('out/'):\n            os.makedirs('out/')\n\n        plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight')\n        cnt += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/f_gan/f_gan_tensorflow.py",
    "content": "import tensorflow as tf\nfrom tensorflow.examples.tutorials.mnist import input_data\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\n\n\nmb_size = 32\nX_dim = 784\nz_dim = 64\nh_dim = 128\nlr = 1e-3\nd_steps = 3\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\n\n\ndef plot(samples):\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    return fig\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)\n    return tf.random_normal(shape=size, stddev=xavier_stddev)\n\n\nX = tf.placeholder(tf.float32, shape=[None, X_dim])\nz = tf.placeholder(tf.float32, shape=[None, z_dim])\n\nD_W1 = tf.Variable(xavier_init([X_dim, h_dim]))\nD_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nD_W2 = tf.Variable(xavier_init([h_dim, 1]))\nD_b2 = tf.Variable(tf.zeros(shape=[1]))\n\nG_W1 = tf.Variable(xavier_init([z_dim, h_dim]))\nG_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nG_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nG_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\ntheta_G = [G_W1, G_W2, G_b1, G_b2]\ntheta_D = [D_W1, D_W2, D_b1, D_b2]\n\n\ndef sample_z(m, n):\n    return np.random.uniform(-1., 1., size=[m, n])\n\n\ndef generator(z):\n    G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1)\n    G_log_prob = tf.matmul(G_h1, G_W2) + G_b2\n    G_prob = tf.nn.sigmoid(G_log_prob)\n    return G_prob\n\n\ndef discriminator(x):\n    D_h1 = tf.nn.relu(tf.matmul(x, D_W1) + D_b1)\n    out = tf.matmul(D_h1, D_W2) + D_b2\n    return out\n\n\nG_sample = generator(z)\n\nD_real = discriminator(X)\nD_fake = discriminator(G_sample)\n\n# Uncomment D_loss and its respective G_loss of your choice\n# ---------------------------------------------------------\n\n\"\"\" Total Variation \"\"\"\n# D_loss = -(tf.reduce_mean(0.5 * tf.nn.tanh(D_real)) -\n#            tf.reduce_mean(0.5 * tf.nn.tanh(D_fake)))\n# G_loss = -tf.reduce_mean(0.5 * tf.nn.tanh(D_fake))\n\n\"\"\" Forward KL \"\"\"\n# D_loss = -(tf.reduce_mean(D_real) - tf.reduce_mean(tf.exp(D_fake - 1)))\n# G_loss = -tf.reduce_mean(tf.exp(D_fake - 1))\n\n\"\"\" Reverse KL \"\"\"\n# D_loss = -(tf.reduce_mean(-tf.exp(D_real)) - tf.reduce_mean(-1 - D_fake))\n# G_loss = -tf.reduce_mean(-1 - D_fake)\n\n\"\"\" Pearson Chi-squared \"\"\"\nD_loss = -(tf.reduce_mean(D_real) - tf.reduce_mean(0.25*D_fake**2 + D_fake))\nG_loss = -tf.reduce_mean(0.25*D_fake**2 + D_fake)\n\n\"\"\" Squared Hellinger \"\"\"\n# D_loss = -(tf.reduce_mean(1 - tf.exp(D_real)) -\n#            tf.reduce_mean((1 - tf.exp(D_fake)) / (tf.exp(D_fake))))\n# G_loss = -tf.reduce_mean((1 - tf.exp(D_fake)) / (tf.exp(D_fake)))\n\n\nD_solver = (tf.train.AdamOptimizer(learning_rate=lr)\n            .minimize(D_loss, var_list=theta_D))\nG_solver = (tf.train.AdamOptimizer(learning_rate=lr)\n            .minimize(G_loss, var_list=theta_G))\n\nsess = tf.Session()\nsess.run(tf.global_variables_initializer())\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\ni = 0\n\nfor it in range(1000000):\n    X_mb, _ = mnist.train.next_batch(mb_size)\n    z_mb = sample_z(mb_size, z_dim)\n\n    _, D_loss_curr = sess.run([D_solver, D_loss], feed_dict={X: X_mb, z: z_mb})\n    _, G_loss_curr = sess.run([G_solver, G_loss], feed_dict={z: z_mb})\n\n    if it % 1000 == 0:\n        print('Iter: {}; D_loss: {:.4}; G_loss: {:.4}'\n              .format(it, D_loss_curr, G_loss_curr))\n\n        samples = sess.run(G_sample, feed_dict={z: sample_z(16, z_dim)})\n\n        fig = plot(samples)\n        plt.savefig('out/{}.png'\n                    .format(str(i).zfill(3)), bbox_inches='tight')\n        i += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/generative_adversarial_parallelization/gap_pytorch.py",
    "content": "import torch\nimport torch.nn\nimport torch.nn.functional as nn\nimport torch.autograd as autograd\nimport torch.optim as optim\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nimport random\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 32\nz_dim = 10\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\ncnt = 0\nlr = 1e-3\nK = 100\n\n\ndef log(x):\n    return torch.log(x + 1e-8)\n\n\nG1_ = torch.nn.Sequential(\n    torch.nn.Linear(z_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, X_dim),\n    torch.nn.Sigmoid()\n)\n\n\nD1_ = torch.nn.Sequential(\n    torch.nn.Linear(X_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, 1),\n    torch.nn.Sigmoid()\n)\n\nG2_ = torch.nn.Sequential(\n    torch.nn.Linear(z_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, X_dim),\n    torch.nn.Sigmoid()\n)\n\n\nD2_ = torch.nn.Sequential(\n    torch.nn.Linear(X_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, 1),\n    torch.nn.Sigmoid()\n)\n\nnets = [G1_, D1_, G2_, D2_]\n\n\ndef reset_grad():\n    for net in nets:\n        net.zero_grad()\n\n\nG1_solver = optim.Adam(G1_.parameters(), lr=lr)\nD1_solver = optim.Adam(D1_.parameters(), lr=lr)\nG2_solver = optim.Adam(G2_.parameters(), lr=lr)\nD2_solver = optim.Adam(D2_.parameters(), lr=lr)\n\nD1 = {'model': D1_, 'solver': D1_solver}\nG1 = {'model': G1_, 'solver': G1_solver}\nD2 = {'model': D2_, 'solver': D2_solver}\nG2 = {'model': G2_, 'solver': G2_solver}\n\nGAN_pairs = [(D1, G1), (D2, G2)]\n\nfor it in range(1000000):\n    # Sample data\n    z = Variable(torch.randn(mb_size, z_dim))\n    X, _ = mnist.train.next_batch(mb_size)\n    X = Variable(torch.from_numpy(X))\n\n    for D, G in GAN_pairs:\n        # Discriminator\n        G_sample = G['model'](z)\n        D_real = D['model'](X)\n        D_fake = D['model'](G_sample)\n\n        D_loss = -torch.mean(log(D_real) + log(1 - D_fake))\n\n        D_loss.backward()\n        D['solver'].step()\n        reset_grad()\n\n        # Generator\n        G_sample = G['model'](z)\n        D_fake = D['model'](G_sample)\n\n        G_loss = -torch.mean(log(D_fake))\n\n        G_loss.backward()\n        G['solver'].step()\n        reset_grad()\n\n    if it != 0 and it % K == 0:\n        # Swap (D, G) pairs\n        new_D1, new_D2 = GAN_pairs[1][0], GAN_pairs[0][0]\n        GAN_pairs = [(new_D1, G1), (new_D2, G2)]\n\n    # Print and plot every now and then\n    if it % 1000 == 0:\n        print('Iter-{}; D_loss: {:.4}; G_loss: {:.4}'\n              .format(it, D_loss.data[0], G_loss.data[0]))\n\n        # Pick G randomly\n        G_rand = random.choice([G1_, G2_])\n        samples = G_rand(z).data.numpy()[:16]\n\n        fig = plt.figure(figsize=(4, 4))\n        gs = gridspec.GridSpec(4, 4)\n        gs.update(wspace=0.05, hspace=0.05)\n\n        for i, sample in enumerate(samples):\n            ax = plt.subplot(gs[i])\n            plt.axis('off')\n            ax.set_xticklabels([])\n            ax.set_yticklabels([])\n            ax.set_aspect('equal')\n            plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n        if not os.path.exists('out/'):\n            os.makedirs('out/')\n\n        plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight')\n        cnt += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/gibbsnet/gibbsnet_pytorch.py",
    "content": "import torch\nimport torch.nn\nimport torch.nn.functional as nn\nimport torch.autograd as autograd\nimport torch.optim as optim\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\nfrom itertools import *\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 16\nz_dim = 100\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 256\ncnt = 0\nlr = 1e-4\nN = 10\n\n\ndef log(x):\n    return torch.log(x + 1e-8)\n\n\n# Inference net (Encoder) Q(z|X)\nQ = torch.nn.Sequential(\n    torch.nn.Linear(X_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, z_dim)\n)\n\n# Generator net (Decoder) P(X|z)\nP = torch.nn.Sequential(\n    torch.nn.Linear(z_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, X_dim),\n    torch.nn.Sigmoid()\n)\n\nD_ = torch.nn.Sequential(\n    torch.nn.Linear(X_dim + z_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, 1),\n    torch.nn.Sigmoid()\n)\n\n\ndef D(X, z):\n    return D_(torch.cat([X, z], 1))\n\n\ndef reset_grad():\n    Q.zero_grad()\n    P.zero_grad()\n    D_.zero_grad()\n\n\nG_solver = optim.Adam(chain(Q.parameters(), P.parameters()), lr=lr)\nD_solver = optim.Adam(D_.parameters(), lr=lr)\n\n\nfor it in range(1000000):\n    # Sample data\n    X, _ = mnist.train.next_batch(mb_size)\n    X = Variable(torch.from_numpy(X))\n\n    # Discriminator\n    z_hat = Q(X)\n\n    # Do N step Gibbs sampling\n    z = Variable(torch.randn(mb_size, z_dim))\n\n    for _ in range(N):\n        z_n = z.clone()\n        X_hat = P(z_n)\n        z = Q(X_hat)\n\n    p_data = D(X, z_hat)\n    p_model = D(X_hat, z_n)\n\n    D_loss = -torch.mean(log(p_data) + log(1 - p_model))\n\n    D_loss.backward(retain_graph=True)\n    D_solver.step()\n    G_solver.step()\n    reset_grad()\n\n    G_loss = -torch.mean(log(p_model) + log(1 - p_data))\n\n    G_loss.backward()\n    G_solver.step()\n    reset_grad()\n\n    # Print and plot every now and then\n    if it % 100 == 0:\n        print('Iter-{}; D_loss: {:.4}; G_loss: {:.4}'\n              .format(it, D_loss.data[0], G_loss.data[0]))\n\n        z = Variable(torch.randn(mb_size, z_dim))\n\n        for _ in range(N):\n            z_n = z.clone()\n            X_hat = P(z_n)\n            z = Q(X_hat)\n\n        samples = X_hat.data.numpy()[:16]\n\n        fig = plt.figure(figsize=(4, 4))\n        gs = gridspec.GridSpec(4, 4)\n        gs.update(wspace=0.05, hspace=0.05)\n\n        for i, sample in enumerate(samples):\n            ax = plt.subplot(gs[i])\n            plt.axis('off')\n            ax.set_xticklabels([])\n            ax.set_yticklabels([])\n            ax.set_aspect('equal')\n            plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n        if not os.path.exists('out/'):\n            os.makedirs('out/')\n\n        plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight')\n        cnt += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/improved_wasserstein_gan/wgan_gp_tensorflow.py",
    "content": "import tensorflow as tf\nfrom tensorflow.examples.tutorials.mnist import input_data\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\n\n\nmb_size = 32\nX_dim = 784\nz_dim = 10\nh_dim = 128\nlam = 10\nn_disc = 5\nlr = 1e-4\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\n\n\ndef plot(samples):\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    return fig\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)\n    return tf.random_normal(shape=size, stddev=xavier_stddev)\n\n\nX = tf.placeholder(tf.float32, shape=[None, X_dim])\n\nD_W1 = tf.Variable(xavier_init([X_dim, h_dim]))\nD_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\n\nD_W2 = tf.Variable(xavier_init([h_dim, 1]))\nD_b2 = tf.Variable(tf.zeros(shape=[1]))\n\ntheta_D = [D_W1, D_W2, D_b1, D_b2]\n\n\nz = tf.placeholder(tf.float32, shape=[None, z_dim])\n\nG_W1 = tf.Variable(xavier_init([z_dim, h_dim]))\nG_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\n\nG_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nG_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\ntheta_G = [G_W1, G_W2, G_b1, G_b2]\n\n\ndef sample_z(m, n):\n    return np.random.uniform(-1., 1., size=[m, n])\n\n\ndef G(z):\n    G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1)\n    G_log_prob = tf.matmul(G_h1, G_W2) + G_b2\n    G_prob = tf.nn.sigmoid(G_log_prob)\n    return G_prob\n\n\ndef D(X):\n    D_h1 = tf.nn.relu(tf.matmul(X, D_W1) + D_b1)\n    out = tf.matmul(D_h1, D_W2) + D_b2\n    return out\n\n\nG_sample = G(z)\nD_real = D(X)\nD_fake = D(G_sample)\n\neps = tf.random_uniform([mb_size, 1], minval=0., maxval=1.)\nX_inter = eps*X + (1. - eps)*G_sample\ngrad = tf.gradients(D(X_inter), [X_inter])[0]\ngrad_norm = tf.sqrt(tf.reduce_sum((grad)**2, axis=1))\ngrad_pen = lam * tf.reduce_mean((grad_norm - 1)**2)\n\nD_loss = tf.reduce_mean(D_fake) - tf.reduce_mean(D_real) + grad_pen\nG_loss = -tf.reduce_mean(D_fake)\n\nD_solver = (tf.train.AdamOptimizer(learning_rate=lr, beta1=0.5)\n            .minimize(D_loss, var_list=theta_D))\nG_solver = (tf.train.AdamOptimizer(learning_rate=lr, beta1=0.5)\n            .minimize(G_loss, var_list=theta_G))\n\nsess = tf.Session()\nsess.run(tf.global_variables_initializer())\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\ni = 0\n\nfor it in range(1000000):\n    for _ in range(n_disc):\n        X_mb, _ = mnist.train.next_batch(mb_size)\n\n        _, D_loss_curr = sess.run(\n            [D_solver, D_loss],\n            feed_dict={X: X_mb, z: sample_z(mb_size, z_dim)}\n        )\n\n    _, G_loss_curr = sess.run(\n        [G_solver, G_loss],\n        feed_dict={z: sample_z(mb_size, z_dim)}\n    )\n\n    if it % 1000 == 0:\n        print('Iter: {}; D loss: {:.4}; G_loss: {:.4}'\n              .format(it, D_loss_curr, G_loss_curr))\n\n        if it % 1000 == 0:\n            samples = sess.run(G_sample, feed_dict={z: sample_z(16, z_dim)})\n\n            fig = plot(samples)\n            plt.savefig('out/{}.png'\n                        .format(str(i).zfill(3)), bbox_inches='tight')\n            i += 1\n            plt.close(fig)\n"
  },
  {
    "path": "GAN/infogan/infogan_pytorch.py",
    "content": "import torch\nimport torch.nn.functional as nn\nimport torch.autograd as autograd\nimport torch.optim as optim\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 32\nZ_dim = 16\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\ncnt = 0\nlr = 1e-3\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / np.sqrt(in_dim / 2.)\n    return Variable(torch.randn(*size) * xavier_stddev, requires_grad=True)\n\n\n\"\"\" ==================== GENERATOR ======================== \"\"\"\n\nWzh = xavier_init(size=[Z_dim + 10, h_dim])\nbzh = Variable(torch.zeros(h_dim), requires_grad=True)\n\nWhx = xavier_init(size=[h_dim, X_dim])\nbhx = Variable(torch.zeros(X_dim), requires_grad=True)\n\n\ndef G(z, c):\n    inputs = torch.cat([z, c], 1)\n    h = nn.relu(inputs @ Wzh + bzh.repeat(inputs.size(0), 1))\n    X = nn.sigmoid(h @ Whx + bhx.repeat(h.size(0), 1))\n    return X\n\n\n\"\"\" ==================== DISCRIMINATOR ======================== \"\"\"\n\nWxh = xavier_init(size=[X_dim, h_dim])\nbxh = Variable(torch.zeros(h_dim), requires_grad=True)\n\nWhy = xavier_init(size=[h_dim, 1])\nbhy = Variable(torch.zeros(1), requires_grad=True)\n\n\ndef D(X):\n    h = nn.relu(X @ Wxh + bxh.repeat(X.size(0), 1))\n    y = nn.sigmoid(h @ Why + bhy.repeat(h.size(0), 1))\n    return y\n\n\n\"\"\" ====================== Q(c|X) ========================== \"\"\"\n\nWqxh = xavier_init(size=[X_dim, h_dim])\nbqxh = Variable(torch.zeros(h_dim), requires_grad=True)\n\nWhc = xavier_init(size=[h_dim, 10])\nbhc = Variable(torch.zeros(10), requires_grad=True)\n\n\ndef Q(X):\n    h = nn.relu(X @ Wqxh + bqxh.repeat(X.size(0), 1))\n    c = nn.softmax(h @ Whc + bhc.repeat(h.size(0), 1))\n    return c\n\n\nG_params = [Wzh, bzh, Whx, bhx]\nD_params = [Wxh, bxh, Why, bhy]\nQ_params = [Wqxh, bqxh, Whc, bhc]\nparams = G_params + D_params + Q_params\n\n\n\"\"\" ===================== TRAINING ======================== \"\"\"\n\n\ndef reset_grad():\n    for p in params:\n        if p.grad is not None:\n            data = p.grad.data\n            p.grad = Variable(data.new().resize_as_(data).zero_())\n\n\nG_solver = optim.Adam(G_params, lr=1e-3)\nD_solver = optim.Adam(D_params, lr=1e-3)\nQ_solver = optim.Adam(G_params + Q_params, lr=1e-3)\n\n\ndef sample_c(size):\n    c = np.random.multinomial(1, 10*[0.1], size=size)\n    c = Variable(torch.from_numpy(c.astype('float32')))\n    return c\n\n\nfor it in range(100000):\n    # Sample data\n    X, _ = mnist.train.next_batch(mb_size)\n    X = Variable(torch.from_numpy(X))\n\n    z = Variable(torch.randn(mb_size, Z_dim))\n    c = sample_c(mb_size)\n\n    # Dicriminator forward-loss-backward-update\n    G_sample = G(z, c)\n    D_real = D(X)\n    D_fake = D(G_sample)\n\n    D_loss = -torch.mean(torch.log(D_real + 1e-8) + torch.log(1 - D_fake + 1e-8))\n\n    D_loss.backward()\n    D_solver.step()\n\n    # Housekeeping - reset gradient\n    reset_grad()\n\n    # Generator forward-loss-backward-update\n    G_sample = G(z, c)\n    D_fake = D(G_sample)\n\n    G_loss = -torch.mean(torch.log(D_fake + 1e-8))\n\n    G_loss.backward()\n    G_solver.step()\n\n    # Housekeeping - reset gradient\n    reset_grad()\n\n    # Q forward-loss-backward-update\n    G_sample = G(z, c)\n    Q_c_given_x = Q(G_sample)\n\n    crossent_loss = torch.mean(-torch.sum(c * torch.log(Q_c_given_x + 1e-8), dim=1))\n    mi_loss = crossent_loss\n\n    mi_loss.backward()\n    Q_solver.step()\n\n    # Housekeeping - reset gradient\n    reset_grad()\n\n    # Print and plot every now and then\n    if it % 1000 == 0:\n        idx = np.random.randint(0, 10)\n        c = np.zeros([mb_size, 10])\n        c[range(mb_size), idx] = 1\n        c = Variable(torch.from_numpy(c.astype('float32')))\n        samples = G(z, c).data.numpy()[:16]\n\n        print('Iter-{}; D_loss: {}; G_loss: {}; Idx: {}'\n              .format(it, D_loss.data.numpy(), G_loss.data.numpy(), idx))\n\n        fig = plt.figure(figsize=(4, 4))\n        gs = gridspec.GridSpec(4, 4)\n        gs.update(wspace=0.05, hspace=0.05)\n\n        for i, sample in enumerate(samples):\n            ax = plt.subplot(gs[i])\n            plt.axis('off')\n            ax.set_xticklabels([])\n            ax.set_yticklabels([])\n            ax.set_aspect('equal')\n            plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n        if not os.path.exists('out/'):\n            os.makedirs('out/')\n\n        plt.savefig('out/{}.png'\n                    .format(str(cnt).zfill(3)), bbox_inches='tight')\n        cnt += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/infogan/infogan_tensorflow.py",
    "content": "import tensorflow as tf\nfrom tensorflow.examples.tutorials.mnist import input_data\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)\n    return tf.random_normal(shape=size, stddev=xavier_stddev)\n\n\nX = tf.placeholder(tf.float32, shape=[None, 784])\n\nD_W1 = tf.Variable(xavier_init([784, 128]))\nD_b1 = tf.Variable(tf.zeros(shape=[128]))\n\nD_W2 = tf.Variable(xavier_init([128, 1]))\nD_b2 = tf.Variable(tf.zeros(shape=[1]))\n\ntheta_D = [D_W1, D_W2, D_b1, D_b2]\n\n\nZ = tf.placeholder(tf.float32, shape=[None, 16])\nc = tf.placeholder(tf.float32, shape=[None, 10])\n\nG_W1 = tf.Variable(xavier_init([26, 256]))\nG_b1 = tf.Variable(tf.zeros(shape=[256]))\n\nG_W2 = tf.Variable(xavier_init([256, 784]))\nG_b2 = tf.Variable(tf.zeros(shape=[784]))\n\ntheta_G = [G_W1, G_W2, G_b1, G_b2]\n\n\nQ_W1 = tf.Variable(xavier_init([784, 128]))\nQ_b1 = tf.Variable(tf.zeros(shape=[128]))\n\nQ_W2 = tf.Variable(xavier_init([128, 10]))\nQ_b2 = tf.Variable(tf.zeros(shape=[10]))\n\ntheta_Q = [Q_W1, Q_W2, Q_b1, Q_b2]\n\n\ndef sample_Z(m, n):\n    return np.random.uniform(-1., 1., size=[m, n])\n\n\ndef sample_c(m):\n    return np.random.multinomial(1, 10*[0.1], size=m)\n\n\ndef generator(z, c):\n    inputs = tf.concat(axis=1, values=[z, c])\n    G_h1 = tf.nn.relu(tf.matmul(inputs, G_W1) + G_b1)\n    G_log_prob = tf.matmul(G_h1, G_W2) + G_b2\n    G_prob = tf.nn.sigmoid(G_log_prob)\n\n    return G_prob\n\n\ndef discriminator(x):\n    D_h1 = tf.nn.relu(tf.matmul(x, D_W1) + D_b1)\n    D_logit = tf.matmul(D_h1, D_W2) + D_b2\n    D_prob = tf.nn.sigmoid(D_logit)\n\n    return D_prob\n\n\ndef Q(x):\n    Q_h1 = tf.nn.relu(tf.matmul(x, Q_W1) + Q_b1)\n    Q_prob = tf.nn.softmax(tf.matmul(Q_h1, Q_W2) + Q_b2)\n\n    return Q_prob\n\n\ndef plot(samples):\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    return fig\n\n\nG_sample = generator(Z, c)\nD_real = discriminator(X)\nD_fake = discriminator(G_sample)\nQ_c_given_x = Q(G_sample)\n\nD_loss = -tf.reduce_mean(tf.log(D_real + 1e-8) + tf.log(1 - D_fake + 1e-8))\nG_loss = -tf.reduce_mean(tf.log(D_fake + 1e-8))\n\ncross_ent = tf.reduce_mean(-tf.reduce_sum(tf.log(Q_c_given_x + 1e-8) * c, 1))\nQ_loss = cross_ent\n\nD_solver = tf.train.AdamOptimizer().minimize(D_loss, var_list=theta_D)\nG_solver = tf.train.AdamOptimizer().minimize(G_loss, var_list=theta_G)\nQ_solver = tf.train.AdamOptimizer().minimize(Q_loss, var_list=theta_G + theta_Q)\n\nmb_size = 32\nZ_dim = 16\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\n\nsess = tf.Session()\nsess.run(tf.global_variables_initializer())\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\ni = 0\n\nfor it in range(1000000):\n    if it % 1000 == 0:\n        Z_noise = sample_Z(16, Z_dim)\n\n        idx = np.random.randint(0, 10)\n        c_noise = np.zeros([16, 10])\n        c_noise[range(16), idx] = 1\n\n        samples = sess.run(G_sample,\n                           feed_dict={Z: Z_noise, c: c_noise})\n\n        fig = plot(samples)\n        plt.savefig('out/{}.png'.format(str(i).zfill(3)), bbox_inches='tight')\n        i += 1\n        plt.close(fig)\n\n    X_mb, _ = mnist.train.next_batch(mb_size)\n    Z_noise = sample_Z(mb_size, Z_dim)\n    c_noise = sample_c(mb_size)\n\n    _, D_loss_curr = sess.run([D_solver, D_loss],\n                              feed_dict={X: X_mb, Z: Z_noise, c: c_noise})\n\n    _, G_loss_curr = sess.run([G_solver, G_loss],\n                              feed_dict={Z: Z_noise, c: c_noise})\n\n    sess.run([Q_solver], feed_dict={Z: Z_noise, c: c_noise})\n\n    if it % 1000 == 0:\n        print('Iter: {}'.format(it))\n        print('D loss: {:.4}'. format(D_loss_curr))\n        print('G_loss: {:.4}'.format(G_loss_curr))\n        print()\n"
  },
  {
    "path": "GAN/least_squares_gan/lsgan_pytorch.py",
    "content": "import torch\nimport torch.nn\nimport torch.nn.functional as nn\nimport torch.autograd as autograd\nimport torch.optim as optim\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 32\nz_dim = 10\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\ncnt = 0\nd_step = 3\nlr = 1e-3\n\n\nG = torch.nn.Sequential(\n    torch.nn.Linear(z_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, X_dim),\n    torch.nn.Sigmoid()\n)\n\n\nD = torch.nn.Sequential(\n    torch.nn.Linear(X_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, 1),\n)\n\n\ndef reset_grad():\n    G.zero_grad()\n    D.zero_grad()\n\n\nG_solver = optim.Adam(G.parameters(), lr=lr)\nD_solver = optim.Adam(D.parameters(), lr=lr)\n\n\nfor it in range(1000000):\n    for _ in range(d_step):\n        # Sample data\n        z = Variable(torch.randn(mb_size, z_dim))\n        X, _ = mnist.train.next_batch(mb_size)\n        X = Variable(torch.from_numpy(X))\n\n        # Dicriminator\n        G_sample = G(z)\n        D_real = D(X)\n        D_fake = D(G_sample)\n\n        D_loss = 0.5 * (torch.mean((D_real - 1)**2) + torch.mean(D_fake**2))\n\n        D_loss.backward()\n        D_solver.step()\n        reset_grad()\n\n    # Generator\n    z = Variable(torch.randn(mb_size, z_dim))\n\n    G_sample = G(z)\n    D_fake = D(G_sample)\n\n    G_loss = 0.5 * torch.mean((D_fake - 1)**2)\n\n    G_loss.backward()\n    G_solver.step()\n    reset_grad()\n\n    # Print and plot every now and then\n    if it % 1000 == 0:\n        print('Iter-{}; D_loss: {:.4}; G_loss: {:.4}'\n              .format(it, D_loss.data[0], G_loss.data[0]))\n\n        samples = G(z).data.numpy()[:16]\n\n        fig = plt.figure(figsize=(4, 4))\n        gs = gridspec.GridSpec(4, 4)\n        gs.update(wspace=0.05, hspace=0.05)\n\n        for i, sample in enumerate(samples):\n            ax = plt.subplot(gs[i])\n            plt.axis('off')\n            ax.set_xticklabels([])\n            ax.set_yticklabels([])\n            ax.set_aspect('equal')\n            plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n        if not os.path.exists('out/'):\n            os.makedirs('out/')\n\n        plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight')\n        cnt += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/least_squares_gan/lsgan_tensorflow.py",
    "content": "import tensorflow as tf\nfrom tensorflow.examples.tutorials.mnist import input_data\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\n\n\nmb_size = 32\nX_dim = 784\nz_dim = 64\nh_dim = 128\nlr = 1e-3\nd_steps = 3\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\n\n\ndef plot(samples):\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    return fig\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)\n    return tf.random_normal(shape=size, stddev=xavier_stddev)\n\n\nX = tf.placeholder(tf.float32, shape=[None, X_dim])\nz = tf.placeholder(tf.float32, shape=[None, z_dim])\n\nD_W1 = tf.Variable(xavier_init([X_dim, h_dim]))\nD_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nD_W2 = tf.Variable(xavier_init([h_dim, 1]))\nD_b2 = tf.Variable(tf.zeros(shape=[1]))\n\nG_W1 = tf.Variable(xavier_init([z_dim, h_dim]))\nG_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nG_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nG_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\ntheta_G = [G_W1, G_W2, G_b1, G_b2]\ntheta_D = [D_W1, D_W2, D_b1, D_b2]\n\n\ndef sample_z(m, n):\n    return np.random.uniform(-1., 1., size=[m, n])\n\n\ndef generator(z):\n    G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1)\n    G_log_prob = tf.matmul(G_h1, G_W2) + G_b2\n    G_prob = tf.nn.sigmoid(G_log_prob)\n    return G_prob\n\n\ndef discriminator(x):\n    D_h1 = tf.nn.relu(tf.matmul(x, D_W1) + D_b1)\n    out = tf.matmul(D_h1, D_W2) + D_b2\n    return out\n\n\nG_sample = generator(z)\n\nD_real = discriminator(X)\nD_fake = discriminator(G_sample)\n\nD_loss = 0.5 * (tf.reduce_mean((D_real - 1)**2) + tf.reduce_mean(D_fake**2))\nG_loss = 0.5 * tf.reduce_mean((D_fake - 1)**2)\n\nD_solver = (tf.train.AdamOptimizer(learning_rate=lr)\n            .minimize(D_loss, var_list=theta_D))\nG_solver = (tf.train.AdamOptimizer(learning_rate=lr)\n            .minimize(G_loss, var_list=theta_G))\n\nsess = tf.Session()\nsess.run(tf.global_variables_initializer())\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\ni = 0\n\nfor it in range(1000000):\n    for _ in range(d_steps):\n        X_mb, _ = mnist.train.next_batch(mb_size)\n        z_mb = sample_z(mb_size, z_dim)\n\n        _, D_loss_curr = sess.run(\n            [D_solver, D_loss],\n            feed_dict={X: X_mb, z: z_mb}\n        )\n\n    X_mb, _ = mnist.train.next_batch(mb_size)\n    z_mb = sample_z(mb_size, z_dim)\n\n    _, G_loss_curr = sess.run(\n        [G_solver, G_loss],\n        feed_dict={X: X_mb, z: sample_z(mb_size, z_dim)}\n    )\n\n    if it % 1000 == 0:\n        print('Iter: {}; D_loss: {:.4}; G_loss: {:.4}'\n              .format(it, D_loss_curr, G_loss_curr))\n\n        samples = sess.run(G_sample, feed_dict={z: sample_z(16, z_dim)})\n\n        fig = plot(samples)\n        plt.savefig('out/{}.png'\n                    .format(str(i).zfill(3)), bbox_inches='tight')\n        i += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/magan/magan_pytorch.py",
    "content": "import torch\nimport torch.nn\nimport torch.nn.functional as nn\nimport torch.autograd as autograd\nimport torch.optim as optim\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 32\nz_dim = 10\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\ncnt = 0\nd_step = 3\nlr = 5e-4\nm = 5\nn_iter = 1000\nn_epoch = 1000\nN = n_iter * mb_size  # N data per epoch\n\n\nG = torch.nn.Sequential(\n    torch.nn.Linear(z_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, X_dim),\n    torch.nn.Sigmoid()\n)\n\n# D is an autoencoder\nD_ = torch.nn.Sequential(\n    torch.nn.Linear(X_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, X_dim),\n)\n\n\n# Energy is the MSE of autoencoder\ndef D(X):\n    X_recon = D_(X)\n    return torch.sum((X - X_recon)**2, 1)\n\n\ndef reset_grad():\n    G.zero_grad()\n    D_.zero_grad()\n\n\nG_solver = optim.Adamax(G.parameters(), lr=lr)\nD_solver = optim.Adamax(D_.parameters(), lr=lr)\n\n\n# Pretrain discriminator\nfor it in range(2*n_iter):\n    X, _ = mnist.train.next_batch(mb_size)\n    X = Variable(torch.from_numpy(X))\n\n    loss = torch.mean(D(X))  # Minimize real samples energy\n\n    loss.backward()\n    D_solver.step()\n    reset_grad()\n\n    if it % 1000 == 0:\n        print('Iter-{}; Pretrained D loss: {:.4}'.format(it, loss.data[0]))\n\n\n# Initial margin, expected energy of real data\nm = torch.mean(D(Variable(torch.from_numpy(mnist.train.images)))).data[0]\ns_z_before = torch.from_numpy(np.array([np.inf], dtype='float32'))\n\n\n# GAN training\nfor t in range(n_epoch):\n    s_x, s_z = torch.zeros(1), torch.zeros(1)\n\n    for it in range(n_iter):\n        # Sample data\n        z = Variable(torch.randn(mb_size, z_dim))\n        X, _ = mnist.train.next_batch(mb_size)\n        X = Variable(torch.from_numpy(X))\n\n        # Dicriminator\n        G_sample = G(z)\n        D_real = D(X)\n        D_fake = D(G_sample)\n\n        D_loss = torch.mean(D_real) + nn.relu(m - torch.mean(D_fake))\n\n        D_loss.backward()\n        D_solver.step()\n\n        # Update real samples statistics\n        s_x += torch.sum(D_real.data)\n\n        reset_grad()\n\n        # Generator\n        z = Variable(torch.randn(mb_size, z_dim))\n        G_sample = G(z)\n        D_fake = D(G_sample)\n\n        G_loss = torch.mean(D_fake)\n\n        G_loss.backward()\n        G_solver.step()\n\n        # Update fake samples statistics\n        s_z += torch.sum(D_fake.data)\n\n        reset_grad()\n\n    # Update margin\n    if (((s_x[0] / N) < m) and (s_x[0] < s_z[0]) and (s_z_before[0] < s_z[0])):\n        m = s_x[0] / N\n\n    s_z_before = s_z\n\n    # Convergence measure\n    Ex = s_x[0] / N\n    Ez = s_z[0] / N\n    L = Ex + np.abs(Ex - Ez)\n\n    # Visualize\n    print('Epoch-{}; m = {:.4}; L = {:.4}'\n          .format(t, m, L))\n\n    samples = G(z).data.numpy()[:16]\n\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    if not os.path.exists('out/'):\n        os.makedirs('out/')\n\n    plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight')\n    cnt += 1\n    plt.close(fig)\n"
  },
  {
    "path": "GAN/magan/magan_tensorflow.py",
    "content": "import tensorflow as tf\nfrom tensorflow.examples.tutorials.mnist import input_data\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\n\n\nmb_size = 32\nX_dim = 784\nz_dim = 64\nh_dim = 128\nlr = 5e-4\nn_iter = 1000\nn_epoch = 1000\nN = n_iter * mb_size  # N data per epoch\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\n\n\ndef plot(samples):\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    return fig\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)\n    return tf.random_normal(shape=size, stddev=xavier_stddev)\n\n\nX = tf.placeholder(tf.float32, shape=[None, X_dim])\nz = tf.placeholder(tf.float32, shape=[None, z_dim])\nm = tf.placeholder(tf.float32)\n\nD_W1 = tf.Variable(xavier_init([X_dim, h_dim]))\nD_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nD_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nD_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\nG_W1 = tf.Variable(xavier_init([z_dim, h_dim]))\nG_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nG_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nG_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\ntheta_G = [G_W1, G_W2, G_b1, G_b2]\ntheta_D = [D_W1, D_W2, D_b1, D_b2]\n\n\ndef sample_z(m, n):\n    return np.random.uniform(-1., 1., size=[m, n])\n\n\ndef G(z):\n    G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1)\n    G_log_prob = tf.matmul(G_h1, G_W2) + G_b2\n    G_prob = tf.nn.sigmoid(G_log_prob)\n    return G_prob\n\n\ndef D(X):\n    D_h1 = tf.nn.relu(tf.matmul(X, D_W1) + D_b1)\n    X_recon = tf.matmul(D_h1, D_W2) + D_b2\n    return tf.reduce_sum((X - X_recon)**2, 1)\n\n\nG_sample = G(z)\n\nD_real = D(X)\nD_fake = D(G_sample)\n\nD_recon_loss = tf.reduce_mean(D_real)\nD_loss = tf.reduce_mean(D_real + tf.maximum(0., m - D_fake))\nG_loss = tf.reduce_mean(D_fake)\n\nD_recon_solver = (tf.train.AdamOptimizer(learning_rate=lr)\n                  .minimize(D_recon_loss, var_list=theta_D))\nD_solver = (tf.train.AdamOptimizer(learning_rate=lr)\n            .minimize(D_loss, var_list=theta_D))\nG_solver = (tf.train.AdamOptimizer(learning_rate=lr)\n            .minimize(G_loss, var_list=theta_G))\n\nsess = tf.Session()\nsess.run(tf.global_variables_initializer())\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\n\n# Pretrain\nfor it in range(2*n_iter):\n    X_mb, _ = mnist.train.next_batch(mb_size)\n\n    _, D_recon_loss_curr = sess.run(\n        [D_recon_solver, D_recon_loss], feed_dict={X: X_mb}\n    )\n\n    if it % 1000 == 0:\n        print('Iter-{}; Pretrained D loss: {:.4}'.format(it, D_recon_loss_curr))\n\n\ni = 0\n# Initial margin, expected energy of real data\nmargin = sess.run(D_recon_loss, feed_dict={X: mnist.train.images})\ns_z_before = np.inf\n\n# GAN training\nfor t in range(n_epoch):\n    s_x, s_z = 0., 0.\n\n    for it in range(n_iter):\n        X_mb, _ = mnist.train.next_batch(mb_size)\n        z_mb = sample_z(mb_size, z_dim)\n\n        _, D_loss_curr, D_real_curr = sess.run(\n            [D_solver, D_loss, D_real], feed_dict={X: X_mb, z: z_mb, m: margin}\n        )\n\n        # Update real samples statistics\n        s_x += np.sum(D_real_curr)\n\n        _, G_loss_curr, D_fake_curr = sess.run(\n            [G_solver, G_loss, D_fake],\n            feed_dict={X: X_mb, z: sample_z(mb_size, z_dim), m: margin}\n        )\n\n        # Update fake samples statistics\n        s_z += np.sum(D_fake_curr)\n\n    # Update margin\n    if (s_x / N < margin) and (s_x < s_z) and (s_z_before < s_z):\n        margin = s_x / N\n\n    s_z_before = s_z\n\n    # Convergence measure\n    Ex = s_x / N\n    Ez = s_z / N\n    L = Ex + np.abs(Ex - Ez)\n\n    # Visualize\n    print('Epoch: {}; m: {:.4}, L: {:.4}'.format(t, margin, L))\n\n    samples = sess.run(G_sample, feed_dict={z: sample_z(16, z_dim)})\n\n    fig = plot(samples)\n    plt.savefig('out/{}.png'\n                .format(str(i).zfill(3)), bbox_inches='tight')\n    i += 1\n    plt.close(fig)\n"
  },
  {
    "path": "GAN/mode_regularized_gan/mode_reg_gan_pytorch.py",
    "content": "import torch\nimport torch.nn\nimport torch.nn.functional as nn\nimport torch.autograd as autograd\nimport torch.optim as optim\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 32\nz_dim = 128\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\ncnt = 0\nlr = 1e-4\nlam1 = 1e-2\nlam2 = 1e-2\n\n\ndef log(x):\n    return torch.log(x + 1e-8)\n\n\nE = torch.nn.Sequential(\n    torch.nn.Linear(X_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, z_dim)\n)\n\nG = torch.nn.Sequential(\n    torch.nn.Linear(z_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, X_dim),\n    torch.nn.Sigmoid()\n)\n\nD = torch.nn.Sequential(\n    torch.nn.Linear(X_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, 1),\n    torch.nn.Sigmoid()\n)\n\n\ndef reset_grad():\n    G.zero_grad()\n    D.zero_grad()\n    E.zero_grad()\n\n\ndef sample_X(size, include_y=False):\n    X, y = mnist.train.next_batch(size)\n    X = Variable(torch.from_numpy(X))\n\n    if include_y:\n        y = np.argmax(y, axis=1).astype(np.int)\n        y = Variable(torch.from_numpy(y))\n        return X, y\n\n    return X\n\n\nE_solver = optim.Adam(E.parameters(), lr=lr)\nG_solver = optim.Adam(G.parameters(), lr=lr)\nD_solver = optim.Adam(D.parameters(), lr=lr)\n\n\nfor it in range(1000000):\n    \"\"\" Discriminator \"\"\"\n    # Sample data\n    X = sample_X(mb_size)\n    z = Variable(torch.randn(mb_size, z_dim))\n\n    # Dicriminator_1 forward-loss-backward-update\n    G_sample = G(z)\n    D_real = D(X)\n    D_fake = D(G_sample)\n\n    D_loss = -torch.mean(log(D_real) + log(1 - D_fake))\n\n    D_loss.backward()\n    D_solver.step()\n\n    # Housekeeping - reset gradient\n    reset_grad()\n\n    \"\"\" Generator \"\"\"\n    # Sample data\n    X = sample_X(mb_size)\n    z = Variable(torch.randn(mb_size, z_dim))\n\n    # Generator forward-loss-backward-update\n    G_sample = G(z)\n    G_sample_reg = G(E(X))\n    D_fake = D(G_sample)\n    D_reg = D(G_sample_reg)\n\n    mse = torch.sum((X - G_sample_reg)**2, 1)\n    reg = torch.mean(lam1 * mse + lam2 * log(D_reg))\n    G_loss = -torch.mean(log(D_fake)) + reg\n\n    G_loss.backward()\n    G_solver.step()\n\n    # Housekeeping - reset gradient\n    reset_grad()\n\n    \"\"\" Encoder \"\"\"\n    # Sample data\n    X = sample_X(mb_size)\n    z = Variable(torch.randn(mb_size, z_dim))\n\n    G_sample_reg = G(E(X))\n    D_reg = D(G_sample_reg)\n\n    mse = torch.sum((X - G_sample_reg)**2, 1)\n    E_loss = torch.mean(lam1 * mse + lam2 * log(D_reg))\n\n    E_loss.backward()\n    E_solver.step()\n\n    # Housekeeping - reset gradient\n    reset_grad()\n\n    # Print and plot every now and then\n    if it % 1000 == 0:\n        print('Iter-{}; D_loss: {}; E_loss: {}; G_loss: {}'\n              .format(it, D_loss.data.numpy(), E_loss.data.numpy(), G_loss.data.numpy()))\n\n        samples = G(z).data.numpy()[:16]\n\n        fig = plt.figure(figsize=(4, 4))\n        gs = gridspec.GridSpec(4, 4)\n        gs.update(wspace=0.05, hspace=0.05)\n\n        for i, sample in enumerate(samples):\n            ax = plt.subplot(gs[i])\n            plt.axis('off')\n            ax.set_xticklabels([])\n            ax.set_yticklabels([])\n            ax.set_aspect('equal')\n            plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n        if not os.path.exists('out/'):\n            os.makedirs('out/')\n\n        plt.savefig('out/{}.png'\n                    .format(str(cnt).zfill(3)), bbox_inches='tight')\n        cnt += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/mode_regularized_gan/mode_reg_gan_tensorflow.py",
    "content": "import tensorflow as tf\nfrom tensorflow.examples.tutorials.mnist import input_data\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\n\n\nmb_size = 32\nX_dim = 784\nz_dim = 10\nh_dim = 128\nlam1 = 1e-2\nlam2 = 1e-2\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\n\n\ndef plot(samples):\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    return fig\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)\n    return tf.random_normal(shape=size, stddev=xavier_stddev)\n\n\ndef log(x):\n    return tf.log(x + 1e-8)\n\n\nX = tf.placeholder(tf.float32, shape=[None, X_dim])\nz = tf.placeholder(tf.float32, shape=[None, z_dim])\n\nE_W1 = tf.Variable(xavier_init([X_dim, h_dim]))\nE_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nE_W2 = tf.Variable(xavier_init([h_dim, z_dim]))\nE_b2 = tf.Variable(tf.zeros(shape=[z_dim]))\n\nD_W1 = tf.Variable(xavier_init([X_dim, h_dim]))\nD_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nD_W2 = tf.Variable(xavier_init([h_dim, 1]))\nD_b2 = tf.Variable(tf.zeros(shape=[1]))\n\nG_W1 = tf.Variable(xavier_init([z_dim, h_dim]))\nG_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\n\nG_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nG_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\ntheta_E = [E_W1, E_W2, E_b1, E_b2]\ntheta_G = [G_W1, G_W2, G_b1, G_b2]\ntheta_D = [D_W1, D_W2, D_b1, D_b2]\n\n\ndef sample_z(m, n):\n    return np.random.uniform(-1., 1., size=[m, n])\n\n\ndef encoder(x):\n    E_h1 = tf.nn.relu(tf.matmul(x, E_W1) + E_b1)\n    out = tf.matmul(E_h1, E_W2) + E_b2\n    return out\n\n\ndef generator(z):\n    G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1)\n    G_log_prob = tf.matmul(G_h1, G_W2) + G_b2\n    G_prob = tf.nn.sigmoid(G_log_prob)\n    return G_prob\n\n\ndef discriminator(x):\n    D_h1 = tf.nn.relu(tf.matmul(x, D_W1) + D_b1)\n    D_log_prob = tf.matmul(D_h1, D_W2) + D_b2\n    D_prob = tf.nn.sigmoid(D_log_prob)\n    return D_prob\n\n\nG_sample = generator(z)\nG_sample_reg = generator(encoder(X))\n\nD_real = discriminator(X)\nD_fake = discriminator(G_sample)\nD_reg = discriminator(G_sample_reg)\n\nmse = tf.reduce_sum((X - G_sample_reg)**2, 1)\n\nD_loss = -tf.reduce_mean(log(D_real) + log(1 - D_fake))\nE_loss = tf.reduce_mean(lam1 * mse + lam2 * log(D_reg))\nG_loss = -tf.reduce_mean(log(D_fake)) + E_loss\n\nE_solver = (tf.train.AdamOptimizer(learning_rate=1e-3)\n            .minimize(E_loss, var_list=theta_E))\nD_solver = (tf.train.AdamOptimizer(learning_rate=1e-3)\n            .minimize(D_loss, var_list=theta_D))\nG_solver = (tf.train.AdamOptimizer(learning_rate=1e-3)\n            .minimize(G_loss, var_list=theta_G))\n\nsess = tf.Session()\nsess.run(tf.global_variables_initializer())\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\ni = 0\n\nfor it in range(1000000):\n    X_mb, _ = mnist.train.next_batch(mb_size)\n\n    _, D_loss_curr = sess.run(\n        [D_solver, D_loss],\n        feed_dict={X: X_mb, z: sample_z(mb_size, z_dim)}\n    )\n\n    _, G_loss_curr = sess.run(\n        [G_solver, G_loss],\n        feed_dict={X: X_mb, z: sample_z(mb_size, z_dim)}\n    )\n\n    _, E_loss_curr = sess.run(\n        [E_solver, E_loss],\n        feed_dict={X: X_mb, z: sample_z(mb_size, z_dim)}\n    )\n\n    if it % 1000 == 0:\n        print('Iter: {}; D_loss: {:.4}; G_loss: {:.4}; E_loss: {:.4}'\n              .format(it, D_loss_curr, G_loss_curr, E_loss_curr))\n\n        samples = sess.run(G_sample, feed_dict={z: sample_z(16, z_dim)})\n\n        fig = plot(samples)\n        plt.savefig('out/{}.png'\n                    .format(str(i).zfill(3)), bbox_inches='tight')\n        i += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/softmax_gan/softmax_gan_pytorch.py",
    "content": "import torch\nimport torch.nn\nimport torch.nn.functional as nn\nimport torch.autograd as autograd\nimport torch.optim as optim\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 16\nz_dim = 10\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\ncnt = 0\nlr = 1e-3\n\n\ndef log(x):\n    return torch.log(x + 1e-8)\n\n\nG = torch.nn.Sequential(\n    torch.nn.Linear(z_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, X_dim),\n    torch.nn.Sigmoid()\n)\n\n\nD = torch.nn.Sequential(\n    torch.nn.Linear(X_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, 1)\n)\n\n\ndef reset_grad():\n    G.zero_grad()\n    D.zero_grad()\n\n\nG_solver = optim.Adam(G.parameters(), lr=lr)\nD_solver = optim.Adam(D.parameters(), lr=lr)\n\n\nD_target = 1./mb_size\nG_target = 1./(mb_size*2)\n\nfor it in range(1000000):\n    # Sample data\n    z = Variable(torch.randn(mb_size, z_dim))\n    X, _ = mnist.train.next_batch(mb_size)\n    X = Variable(torch.from_numpy(X))\n\n    G_sample = G(z)\n    D_real = D(X)\n    D_fake = D(G_sample)\n\n    # Partition func.\n    Z = torch.sum(torch.exp(-D_real)) + torch.sum(torch.exp(-D_fake))\n\n    # Dicriminator\n    D_loss = torch.sum(D_target * D_real) + log(Z)\n\n    D_loss.backward(retain_graph=True)\n    D_solver.step()\n    reset_grad()\n\n    # Generator\n    G_loss = torch.sum(G_target * D_real) + torch.sum(G_target * D_fake) + log(Z)\n\n    G_loss.backward()\n    G_solver.step()\n    reset_grad()\n\n    # Print and plot every now and then\n    if it % 1000 == 0:\n        print('Iter-{}; D_loss: {:.4}; G_loss: {:.4}'\n              .format(it, D_loss.data[0], G_loss.data[0]))\n\n        samples = G(z).data.numpy()[:16]\n\n        fig = plt.figure(figsize=(4, 4))\n        gs = gridspec.GridSpec(4, 4)\n        gs.update(wspace=0.05, hspace=0.05)\n\n        for i, sample in enumerate(samples):\n            ax = plt.subplot(gs[i])\n            plt.axis('off')\n            ax.set_xticklabels([])\n            ax.set_yticklabels([])\n            ax.set_aspect('equal')\n            plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n        if not os.path.exists('out/'):\n            os.makedirs('out/')\n\n        plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight')\n        cnt += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/softmax_gan/softmax_gan_tensorflow.py",
    "content": "import tensorflow as tf\nfrom tensorflow.examples.tutorials.mnist import input_data\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\n\n\nmb_size = 32\nX_dim = 784\nz_dim = 64\nh_dim = 128\nlr = 1e-3\nd_steps = 3\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\n\n\ndef plot(samples):\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    return fig\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)\n    return tf.random_normal(shape=size, stddev=xavier_stddev)\n\n\ndef log(x):\n    return tf.log(x + 1e-8)\n\n\nX = tf.placeholder(tf.float32, shape=[None, X_dim])\nz = tf.placeholder(tf.float32, shape=[None, z_dim])\n\nD_W1 = tf.Variable(xavier_init([X_dim, h_dim]))\nD_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nD_W2 = tf.Variable(xavier_init([h_dim, 1]))\nD_b2 = tf.Variable(tf.zeros(shape=[1]))\n\nG_W1 = tf.Variable(xavier_init([z_dim, h_dim]))\nG_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nG_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nG_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\ntheta_G = [G_W1, G_W2, G_b1, G_b2]\ntheta_D = [D_W1, D_W2, D_b1, D_b2]\n\n\ndef sample_z(m, n):\n    return np.random.uniform(-1., 1., size=[m, n])\n\n\ndef G(z):\n    G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1)\n    G_log_prob = tf.matmul(G_h1, G_W2) + G_b2\n    G_prob = tf.nn.sigmoid(G_log_prob)\n    return G_prob\n\n\ndef D(X):\n    D_h1 = tf.nn.relu(tf.matmul(X, D_W1) + D_b1)\n    out = tf.matmul(D_h1, D_W2) + D_b2\n    return out\n\n\nG_sample = G(z)\n\nD_real = D(X)\nD_fake = D(G_sample)\n\nD_target = 1./mb_size\nG_target = 1./(mb_size*2)\n\nZ = tf.reduce_sum(tf.exp(-D_real)) + tf.reduce_sum(tf.exp(-D_fake))\n\nD_loss = tf.reduce_sum(D_target * D_real) + log(Z)\nG_loss = tf.reduce_sum(G_target * D_real) + tf.reduce_sum(G_target * D_fake) + log(Z)\n\nD_solver = (tf.train.AdamOptimizer(learning_rate=lr)\n            .minimize(D_loss, var_list=theta_D))\nG_solver = (tf.train.AdamOptimizer(learning_rate=lr)\n            .minimize(G_loss, var_list=theta_G))\n\nsess = tf.Session()\nsess.run(tf.global_variables_initializer())\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\ni = 0\n\nfor it in range(1000000):\n    X_mb, _ = mnist.train.next_batch(mb_size)\n    z_mb = sample_z(mb_size, z_dim)\n\n    _, D_loss_curr = sess.run(\n        [D_solver, D_loss], feed_dict={X: X_mb, z: z_mb}\n    )\n\n    _, G_loss_curr = sess.run(\n        [G_solver, G_loss], feed_dict={X: X_mb, z: z_mb}\n    )\n\n    if it % 1000 == 0:\n        print('Iter: {}; D_loss: {:.4}; G_loss: {:.4}'\n              .format(it, D_loss_curr, G_loss_curr))\n\n        samples = sess.run(G_sample, feed_dict={z: sample_z(16, z_dim)})\n\n        fig = plot(samples)\n        plt.savefig('out/{}.png'\n                    .format(str(i).zfill(3)), bbox_inches='tight')\n        i += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/vanilla_gan/gan_pytorch.py",
    "content": "import torch\nimport torch.nn.functional as nn\nimport torch.autograd as autograd\nimport torch.optim as optim\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 64\nZ_dim = 100\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\nc = 0\nlr = 1e-3\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / np.sqrt(in_dim / 2.)\n    return Variable(torch.randn(*size) * xavier_stddev, requires_grad=True)\n\n\n\"\"\" ==================== GENERATOR ======================== \"\"\"\n\nWzh = xavier_init(size=[Z_dim, h_dim])\nbzh = Variable(torch.zeros(h_dim), requires_grad=True)\n\nWhx = xavier_init(size=[h_dim, X_dim])\nbhx = Variable(torch.zeros(X_dim), requires_grad=True)\n\n\ndef G(z):\n    h = nn.relu(z @ Wzh + bzh.repeat(z.size(0), 1))\n    X = nn.sigmoid(h @ Whx + bhx.repeat(h.size(0), 1))\n    return X\n\n\n\"\"\" ==================== DISCRIMINATOR ======================== \"\"\"\n\nWxh = xavier_init(size=[X_dim, h_dim])\nbxh = Variable(torch.zeros(h_dim), requires_grad=True)\n\nWhy = xavier_init(size=[h_dim, 1])\nbhy = Variable(torch.zeros(1), requires_grad=True)\n\n\ndef D(X):\n    h = nn.relu(X @ Wxh + bxh.repeat(X.size(0), 1))\n    y = nn.sigmoid(h @ Why + bhy.repeat(h.size(0), 1))\n    return y\n\n\nG_params = [Wzh, bzh, Whx, bhx]\nD_params = [Wxh, bxh, Why, bhy]\nparams = G_params + D_params\n\n\n\"\"\" ===================== TRAINING ======================== \"\"\"\n\n\ndef reset_grad():\n    for p in params:\n        if p.grad is not None:\n            data = p.grad.data\n            p.grad = Variable(data.new().resize_as_(data).zero_())\n\n\nG_solver = optim.Adam(G_params, lr=1e-3)\nD_solver = optim.Adam(D_params, lr=1e-3)\n\nones_label = Variable(torch.ones(mb_size, 1))\nzeros_label = Variable(torch.zeros(mb_size, 1))\n\n\nfor it in range(100000):\n    # Sample data\n    z = Variable(torch.randn(mb_size, Z_dim))\n    X, _ = mnist.train.next_batch(mb_size)\n    X = Variable(torch.from_numpy(X))\n\n    # Dicriminator forward-loss-backward-update\n    G_sample = G(z)\n    D_real = D(X)\n    D_fake = D(G_sample)\n\n    D_loss_real = nn.binary_cross_entropy(D_real, ones_label)\n    D_loss_fake = nn.binary_cross_entropy(D_fake, zeros_label)\n    D_loss = D_loss_real + D_loss_fake\n\n    D_loss.backward()\n    D_solver.step()\n\n    # Housekeeping - reset gradient\n    reset_grad()\n\n    # Generator forward-loss-backward-update\n    z = Variable(torch.randn(mb_size, Z_dim))\n    G_sample = G(z)\n    D_fake = D(G_sample)\n\n    G_loss = nn.binary_cross_entropy(D_fake, ones_label)\n\n    G_loss.backward()\n    G_solver.step()\n\n    # Housekeeping - reset gradient\n    reset_grad()\n\n    # Print and plot every now and then\n    if it % 1000 == 0:\n        print('Iter-{}; D_loss: {}; G_loss: {}'.format(it, D_loss.data.numpy(), G_loss.data.numpy()))\n\n        samples = G(z).data.numpy()[:16]\n\n        fig = plt.figure(figsize=(4, 4))\n        gs = gridspec.GridSpec(4, 4)\n        gs.update(wspace=0.05, hspace=0.05)\n\n        for i, sample in enumerate(samples):\n            ax = plt.subplot(gs[i])\n            plt.axis('off')\n            ax.set_xticklabels([])\n            ax.set_yticklabels([])\n            ax.set_aspect('equal')\n            plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n        if not os.path.exists('out/'):\n            os.makedirs('out/')\n\n        plt.savefig('out/{}.png'.format(str(c).zfill(3)), bbox_inches='tight')\n        c += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/vanilla_gan/gan_tensorflow.py",
    "content": "import tensorflow as tf\nfrom tensorflow.examples.tutorials.mnist import input_data\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)\n    return tf.random_normal(shape=size, stddev=xavier_stddev)\n\n\nX = tf.placeholder(tf.float32, shape=[None, 784])\n\nD_W1 = tf.Variable(xavier_init([784, 128]))\nD_b1 = tf.Variable(tf.zeros(shape=[128]))\n\nD_W2 = tf.Variable(xavier_init([128, 1]))\nD_b2 = tf.Variable(tf.zeros(shape=[1]))\n\ntheta_D = [D_W1, D_W2, D_b1, D_b2]\n\n\nZ = tf.placeholder(tf.float32, shape=[None, 100])\n\nG_W1 = tf.Variable(xavier_init([100, 128]))\nG_b1 = tf.Variable(tf.zeros(shape=[128]))\n\nG_W2 = tf.Variable(xavier_init([128, 784]))\nG_b2 = tf.Variable(tf.zeros(shape=[784]))\n\ntheta_G = [G_W1, G_W2, G_b1, G_b2]\n\n\ndef sample_Z(m, n):\n    return np.random.uniform(-1., 1., size=[m, n])\n\n\ndef generator(z):\n    G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1)\n    G_log_prob = tf.matmul(G_h1, G_W2) + G_b2\n    G_prob = tf.nn.sigmoid(G_log_prob)\n\n    return G_prob\n\n\ndef discriminator(x):\n    D_h1 = tf.nn.relu(tf.matmul(x, D_W1) + D_b1)\n    D_logit = tf.matmul(D_h1, D_W2) + D_b2\n    D_prob = tf.nn.sigmoid(D_logit)\n\n    return D_prob, D_logit\n\n\ndef plot(samples):\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    return fig\n\n\nG_sample = generator(Z)\nD_real, D_logit_real = discriminator(X)\nD_fake, D_logit_fake = discriminator(G_sample)\n\n# D_loss = -tf.reduce_mean(tf.log(D_real) + tf.log(1. - D_fake))\n# G_loss = -tf.reduce_mean(tf.log(D_fake))\n\n# Alternative losses:\n# -------------------\nD_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_real, labels=tf.ones_like(D_logit_real)))\nD_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_fake, labels=tf.zeros_like(D_logit_fake)))\nD_loss = D_loss_real + D_loss_fake\nG_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_fake, labels=tf.ones_like(D_logit_fake)))\n\nD_solver = tf.train.AdamOptimizer().minimize(D_loss, var_list=theta_D)\nG_solver = tf.train.AdamOptimizer().minimize(G_loss, var_list=theta_G)\n\nmb_size = 128\nZ_dim = 100\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\n\nsess = tf.Session()\nsess.run(tf.global_variables_initializer())\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\ni = 0\n\nfor it in range(1000000):\n    if it % 1000 == 0:\n        samples = sess.run(G_sample, feed_dict={Z: sample_Z(16, Z_dim)})\n\n        fig = plot(samples)\n        plt.savefig('out/{}.png'.format(str(i).zfill(3)), bbox_inches='tight')\n        i += 1\n        plt.close(fig)\n\n    X_mb, _ = mnist.train.next_batch(mb_size)\n\n    _, D_loss_curr = sess.run([D_solver, D_loss], feed_dict={X: X_mb, Z: sample_Z(mb_size, Z_dim)})\n    _, G_loss_curr = sess.run([G_solver, G_loss], feed_dict={Z: sample_Z(mb_size, Z_dim)})\n\n    if it % 1000 == 0:\n        print('Iter: {}'.format(it))\n        print('D loss: {:.4}'. format(D_loss_curr))\n        print('G_loss: {:.4}'.format(G_loss_curr))\n        print()\n"
  },
  {
    "path": "GAN/wasserstein_gan/wgan_pytorch.py",
    "content": "import torch\nimport torch.nn\nimport torch.nn.functional as nn\nimport torch.autograd as autograd\nimport torch.optim as optim\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 32\nz_dim = 10\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\ncnt = 0\nlr = 1e-4\n\n\nG = torch.nn.Sequential(\n    torch.nn.Linear(z_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, X_dim),\n    torch.nn.Sigmoid()\n)\n\n\nD = torch.nn.Sequential(\n    torch.nn.Linear(X_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, 1),\n)\n\n\ndef reset_grad():\n    G.zero_grad()\n    D.zero_grad()\n\n\nG_solver = optim.RMSprop(G.parameters(), lr=lr)\nD_solver = optim.RMSprop(D.parameters(), lr=lr)\n\n\nfor it in range(1000000):\n    for _ in range(5):\n        # Sample data\n        z = Variable(torch.randn(mb_size, z_dim))\n        X, _ = mnist.train.next_batch(mb_size)\n        X = Variable(torch.from_numpy(X))\n\n        # Dicriminator forward-loss-backward-update\n        G_sample = G(z)\n        D_real = D(X)\n        D_fake = D(G_sample)\n\n        D_loss = -(torch.mean(D_real) - torch.mean(D_fake))\n\n        D_loss.backward()\n        D_solver.step()\n\n        # Weight clipping\n        for p in D.parameters():\n            p.data.clamp_(-0.01, 0.01)\n\n        # Housekeeping - reset gradient\n        reset_grad()\n\n    # Generator forward-loss-backward-update\n    X, _ = mnist.train.next_batch(mb_size)\n    X = Variable(torch.from_numpy(X))\n    z = Variable(torch.randn(mb_size, z_dim))\n\n    G_sample = G(z)\n    D_fake = D(G_sample)\n\n    G_loss = -torch.mean(D_fake)\n\n    G_loss.backward()\n    G_solver.step()\n\n    # Housekeeping - reset gradient\n    reset_grad()\n\n    # Print and plot every now and then\n    if it % 1000 == 0:\n        print('Iter-{}; D_loss: {}; G_loss: {}'\n              .format(it, D_loss.data.numpy(), G_loss.data.numpy()))\n\n        samples = G(z).data.numpy()[:16]\n\n        fig = plt.figure(figsize=(4, 4))\n        gs = gridspec.GridSpec(4, 4)\n        gs.update(wspace=0.05, hspace=0.05)\n\n        for i, sample in enumerate(samples):\n            ax = plt.subplot(gs[i])\n            plt.axis('off')\n            ax.set_xticklabels([])\n            ax.set_yticklabels([])\n            ax.set_aspect('equal')\n            plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n        if not os.path.exists('out/'):\n            os.makedirs('out/')\n\n        plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight')\n        cnt += 1\n        plt.close(fig)\n"
  },
  {
    "path": "GAN/wasserstein_gan/wgan_tensorflow.py",
    "content": "import tensorflow as tf\nfrom tensorflow.examples.tutorials.mnist import input_data\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\n\n\nmb_size = 32\nX_dim = 784\nz_dim = 10\nh_dim = 128\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\n\n\ndef plot(samples):\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    return fig\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)\n    return tf.random_normal(shape=size, stddev=xavier_stddev)\n\n\nX = tf.placeholder(tf.float32, shape=[None, X_dim])\n\nD_W1 = tf.Variable(xavier_init([X_dim, h_dim]))\nD_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\n\nD_W2 = tf.Variable(xavier_init([h_dim, 1]))\nD_b2 = tf.Variable(tf.zeros(shape=[1]))\n\ntheta_D = [D_W1, D_W2, D_b1, D_b2]\n\n\nz = tf.placeholder(tf.float32, shape=[None, z_dim])\n\nG_W1 = tf.Variable(xavier_init([z_dim, h_dim]))\nG_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\n\nG_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nG_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\ntheta_G = [G_W1, G_W2, G_b1, G_b2]\n\n\ndef sample_z(m, n):\n    return np.random.uniform(-1., 1., size=[m, n])\n\n\ndef generator(z):\n    G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1)\n    G_log_prob = tf.matmul(G_h1, G_W2) + G_b2\n    G_prob = tf.nn.sigmoid(G_log_prob)\n    return G_prob\n\n\ndef discriminator(x):\n    D_h1 = tf.nn.relu(tf.matmul(x, D_W1) + D_b1)\n    out = tf.matmul(D_h1, D_W2) + D_b2\n    return out\n\n\nG_sample = generator(z)\nD_real = discriminator(X)\nD_fake = discriminator(G_sample)\n\nD_loss = tf.reduce_mean(D_real) - tf.reduce_mean(D_fake)\nG_loss = -tf.reduce_mean(D_fake)\n\nD_solver = (tf.train.RMSPropOptimizer(learning_rate=1e-4)\n            .minimize(-D_loss, var_list=theta_D))\nG_solver = (tf.train.RMSPropOptimizer(learning_rate=1e-4)\n            .minimize(G_loss, var_list=theta_G))\n\nclip_D = [p.assign(tf.clip_by_value(p, -0.01, 0.01)) for p in theta_D]\n\nsess = tf.Session()\nsess.run(tf.global_variables_initializer())\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\ni = 0\n\nfor it in range(1000000):\n    for _ in range(5):\n        X_mb, _ = mnist.train.next_batch(mb_size)\n\n        _, D_loss_curr, _ = sess.run(\n            [D_solver, D_loss, clip_D],\n            feed_dict={X: X_mb, z: sample_z(mb_size, z_dim)}\n        )\n\n    _, G_loss_curr = sess.run(\n        [G_solver, G_loss],\n        feed_dict={z: sample_z(mb_size, z_dim)}\n    )\n\n    if it % 100 == 0:\n        print('Iter: {}; D loss: {:.4}; G_loss: {:.4}'\n              .format(it, D_loss_curr, G_loss_curr))\n\n        if it % 1000 == 0:\n            samples = sess.run(G_sample, feed_dict={z: sample_z(16, z_dim)})\n\n            fig = plot(samples)\n            plt.savefig('out/{}.png'\n                        .format(str(i).zfill(3)), bbox_inches='tight')\n            i += 1\n            plt.close(fig)\n"
  },
  {
    "path": "HelmholtzMachine/README.md",
    "content": "# Helmholtz Machines\n\nImplementation of (Binary) Helmholtz Machines.\n\n## Disclaimer\nCurrently the results is not that good. However, it might be useful to be used to gain intuition of Wake-Sleep Algorithm.\n"
  },
  {
    "path": "HelmholtzMachine/vanilla_HM/helmholtz.py",
    "content": "\"\"\"\nOne layer Binary Helmholtz Machine\n==================================\n\"\"\"\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\n\nmb_size = 16\nh_dim = 36\n\n# Recognition/inference weight\nR = np.random.randn(X_dim, h_dim) * 0.001\n# Generative weight\nW = np.random.randn(h_dim, X_dim) * 0.001\n# Generative bias of hidden variables\nB = np.random.randn(h_dim) * 0.001\n\n\ndef sigm(x):\n    return 1/(1 + np.exp(-x))\n\n\ndef infer(X):\n    # mb_size x x_dim -> mb_size x h_dim\n    return sigm(X @ R)\n\n\ndef generate(H):\n    # mb_size x h_dim -> mb_size x x_dim\n    return sigm(H @ W)\n\n\n# Wake-Sleep Algorithm\n# --------------------\nalpha = 0.1\n\nfor t in range(1, 1001):\n    # ----------\n    # Wake phase\n    # ----------\n\n    # Upward pass\n    X_mb = (mnist.train.next_batch(mb_size)[0] > 0.5).astype(np.float)\n    H = np.random.binomial(n=1, p=infer(X_mb))\n\n    # Downward pass\n    H_prime = sigm(B)\n    V = generate(H)\n\n    # Compute gradient\n    dB = H - H_prime\n    dW = np.array([np.outer(H[i], X_mb[i] - V[i]) for i in range(mb_size)])\n\n    # Update generative weight\n    B += (alpha/t) * np.mean(dB, axis=0)\n    W += (alpha/t) * np.mean(dW, axis=0)\n\n    # -----------\n    # Sleep phase\n    # -----------\n\n    # Downward pass\n    H_mb = np.random.binomial(n=1, p=sigm(B))\n    V = np.random.binomial(n=1, p=generate(H_mb))\n\n    # Upward pass\n    H = infer(V)\n\n    # Compute gradient\n    dR = np.array([np.outer(V, H_mb[i] - H[i]) for i in range(mb_size)])\n\n    # Update recognition weight\n    R += (alpha/t) * np.mean(dR, axis=0)\n\n\n# Visualization\n# -------------\n\ndef plot(samples, size, name):\n    size = int(size)\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(size, size), cmap='Greys_r')\n\n    plt.savefig('out/{}.png'.format(name), bbox_inches='tight')\n    plt.close(fig)\n\n\nX = (mnist.test.next_batch(mb_size)[0] > 0.5).astype(np.float)\n\nH = np.random.binomial(n=1, p=infer(X))\nplot(H, np.sqrt(h_dim), 'H')\n\nX_recon = np.random.binomial(n=1, p=generate(H))\nplot(X_recon, np.sqrt(X_dim), 'V')\n"
  },
  {
    "path": "LICENSE",
    "content": "This is free and unencumbered software released into the public domain.\n\nAnyone is free to copy, modify, publish, use, compile, sell, or\ndistribute this software, either in source code form or as a compiled\nbinary, for any purpose, commercial or non-commercial, and by any\nmeans.\n\nIn jurisdictions that recognize copyright laws, the author or authors\nof this software dedicate any and all copyright interest in the\nsoftware to the public domain. We make this dedication for the benefit\nof the public at large and to the detriment of our heirs and\nsuccessors. We intend this dedication to be an overt act of\nrelinquishment in perpetuity of all present and future rights to this\nsoftware under copyright law.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\nIN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR\nOTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,\nARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR\nOTHER DEALINGS IN THE SOFTWARE.\n\nFor more information, please refer to <http://unlicense.org>\n"
  },
  {
    "path": "RBM/README.md",
    "content": "# Restricted Boltzmann Machines\n\nImplementation of (Binary) Restricted Boltzmann Machines (RBM). Hidden and visible variables are Bernoulli\n\n## Algorithm List\n1. RBM with Contrastive Divergence (CD)\n2. RBM with Persistent Contrastive Divergence (PCD)\n"
  },
  {
    "path": "RBM/rbm_binary_cd.py",
    "content": "import numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\nmnist = input_data.read_data_sets('../MNIST_data', one_hot=True)\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\n\nmb_size = 16\nh_dim = 36\n\nW = np.random.randn(X_dim, h_dim) * 0.001\na = np.random.randn(h_dim) * 0.001\nb = np.random.randn(X_dim) * 0.001\n\n\ndef sigm(x):\n    return 1/(1 + np.exp(-x))\n\n\ndef infer(X):\n    # mb_size x x_dim -> mb_size x h_dim\n    return sigm(X @ W)\n\n\ndef generate(H):\n    # mb_size x h_dim -> mb_size x x_dim\n    return sigm(H @ W.T)\n\n\n# Contrastive Divergence\n# ----------------------\n# Approximate the log partition gradient Gibbs sampling\n\nalpha = 0.1\nK = 10  # Num. of Gibbs sampling step\n\nfor t in range(1, 1001):\n    X_mb = (mnist.train.next_batch(mb_size)[0] > 0.5).astype(np.float)\n    g = 0\n    g_a = 0\n    g_b = 0\n\n    for v in X_mb:\n        # E[h|v,W]\n        h = infer(v)\n\n        # Gibbs sampling steps\n        # --------------------\n        v_prime = np.copy(v)\n\n        for k in range(K):\n            # h ~ p(h|v,W)\n            h_prime = np.random.binomial(n=1, p=infer(v_prime))\n            # v ~ p(v|h,W)\n            v_prime = np.random.binomial(n=1, p=generate(h_prime))\n\n        # E[h|v',W]\n        h_prime = infer(v_prime)\n\n        # Compute data gradient\n        grad_w = np.outer(v, h) - np.outer(v_prime, h_prime)\n        grad_a = h - h_prime\n        grad_b = v - v_prime\n\n        # Accumulate minibatch gradient\n        g += grad_w\n        g_a += grad_a\n        g_b += grad_b\n\n    # Monte carlo gradient\n    g *= 1 / mb_size\n    g_a *= 1 / mb_size\n    g_b *= 1 / mb_size\n\n    # Update to maximize\n    W += alpha * g\n    a += alpha * g_a\n    b += alpha * g_b\n\n\n# Visualization\n# -------------\n\ndef plot(samples, size, name):\n    size = int(size)\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(size, size), cmap='Greys_r')\n\n    plt.savefig('out/{}.png'.format(name), bbox_inches='tight')\n    plt.close(fig)\n\n\nX = (mnist.test.next_batch(mb_size)[0] > 0.5).astype(np.float)\n\nH = np.random.binomial(n=1, p=infer(X))\nplot(H, np.sqrt(h_dim), 'H')\n\nX_recon = (generate(H) > 0.5).astype(np.float)\nplot(X_recon, np.sqrt(X_dim), 'V')\n"
  },
  {
    "path": "RBM/rbm_binary_pcd.py",
    "content": "import numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\nmnist = input_data.read_data_sets('../MNIST_data', one_hot=True)\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\n\nmb_size = 16\nh_dim = 36\n\nW = np.random.randn(X_dim, h_dim) * 0.001\n\n\ndef sigm(x):\n    return 1/(1 + np.exp(-x))\n\n\ndef infer(X):\n    # mb_size x x_dim -> mb_size x h_dim\n    return sigm(X @ W)\n\n\ndef generate(H):\n    # mb_size x h_dim -> mb_size x x_dim\n    return sigm(H @ W.T)\n\n\n# Persistent Contrastive Divergence\n# ---------------------------------\n# Approximate the log partition gradient using single step Gibbs sampling\n\nalpha = 0.1\nK = 10  # Num. of MC iteration\n\n# Initialize the markov chain\nV_s = sigm(np.random.randn(mb_size, X_dim))\nH_s = np.random.binomial(n=1, p=0.5, size=[mb_size, h_dim])\n\nfor t in range(1, 1001):\n    X_mb = (mnist.train.next_batch(mb_size)[0] > 0.5).astype(np.float)\n    g = 0\n\n    Mu = infer(X_mb)\n\n    # Gibbs sampling step\n    # -------------------\n    for i, v_s in enumerate(V_s):\n        for k in range(K):\n            # h ~ p(h|v,W)\n            h_prime = np.random.binomial(n=1, p=infer(v_s))\n            # v ~ p(v|h,W)\n            v_prime = np.random.binomial(n=1, p=generate(h_prime))\n\n        # Replace with new sample\n        V_s[i] = v_prime\n        H_s[i] = h_prime\n\n    # Compute average gradient\n    left = np.array([np.outer(X_mb[i], Mu[i]) for i in range(mb_size)])\n    right = np.array([np.outer(V_s[i], H_s[i]) for i in range(mb_size)])\n    g = np.mean(left, axis=0) - np.mean(right, axis=0)\n\n    # Update\n    W += alpha * g  # Maximize likelihood\n\n\n# Visualization\n# -------------\n\ndef plot(samples, size, name):\n    size = int(size)\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(size, size), cmap='Greys_r')\n\n    plt.savefig('out/{}.png'.format(name), bbox_inches='tight')\n    plt.close(fig)\n\n\nX = (mnist.test.next_batch(mb_size)[0] > 0.5).astype(np.float)\n\nH = np.random.binomial(n=1, p=infer(X))\nplot(H, np.sqrt(h_dim), 'H')\n\nX_recon = (generate(H) > 0.5).astype(np.float)\nplot(X_recon, np.sqrt(X_dim), 'V')\n"
  },
  {
    "path": "README.md",
    "content": "# Generative Models\nCollection of generative models, e.g. GAN, VAE in Pytorch and Tensorflow.\nAlso present here are RBM and Helmholtz Machine.\n\n## Note:\nGenerated samples will be stored in `GAN/{gan_model}/out` (or `VAE/{vae_model}/out`, etc) directory during training.\n\n## What's in it?\n\n#### Generative Adversarial Nets (GAN)\n  1. [Vanilla GAN](https://arxiv.org/abs/1406.2661)\n  2. [Conditional GAN](https://arxiv.org/abs/1411.1784)\n  3. [InfoGAN](https://arxiv.org/abs/1606.03657)\n  4. [Wasserstein GAN](https://arxiv.org/abs/1701.07875)\n  5. [Mode Regularized GAN](https://arxiv.org/abs/1612.02136)\n  6. [Coupled GAN](https://arxiv.org/abs/1606.07536)\n  7. [Auxiliary Classifier GAN](https://arxiv.org/abs/1610.09585)\n  8. [Least Squares GAN](https://arxiv.org/abs/1611.04076v2)\n  9. [Boundary Seeking GAN](https://arxiv.org/abs/1702.08431)\n  10. [Energy Based GAN](https://arxiv.org/abs/1609.03126)\n  11. [f-GAN](https://arxiv.org/abs/1606.00709)\n  12. [Generative Adversarial Parallelization](https://arxiv.org/abs/1612.04021)\n  13. [DiscoGAN](https://arxiv.org/abs/1703.05192)\n  14. [Adversarial Feature Learning](https://arxiv.org/abs/1605.09782) & [Adversarially Learned Inference](https://arxiv.org/abs/1606.00704)\n  15. [Boundary Equilibrium GAN](https://arxiv.org/abs/1703.10717)\n  16. [Improved Training for Wasserstein GAN](https://arxiv.org/abs/1704.00028)\n  17. [DualGAN](https://arxiv.org/abs/1704.02510)\n  18. [MAGAN: Margin Adaptation for GAN](https://arxiv.org/abs/1704.03817)\n  19. [Softmax GAN](https://arxiv.org/abs/1704.06191)\n  20. [GibbsNet](https://papers.nips.cc/paper/7094-gibbsnet-iterative-adversarial-inference-for-deep-graphical-models.pdf)\n\n#### Variational Autoencoder (VAE)\n  1. [Vanilla VAE](https://arxiv.org/abs/1312.6114)\n  2. [Conditional VAE](https://arxiv.org/abs/1406.5298)\n  3. [Denoising VAE](https://arxiv.org/abs/1511.06406)\n  4. [Adversarial Autoencoder](https://arxiv.org/abs/1511.05644)\n  5. [Adversarial Variational Bayes](https://arxiv.org/abs/1701.04722)\n\n#### Restricted Boltzmann Machine (RBM)\n  1. [Binary RBM with Contrastive Divergence](http://www.cs.toronto.edu/~fritz/absps/cdmiguel.pdf)\n  2. [Binary RBM with Persistent Contrastive Divergence](http://www.cs.toronto.edu/~tijmen/pcd/pcd.pdf)\n\n#### Helmholtz Machine\n  1. [Binary Helmholtz Machine with Wake-Sleep Algorithm](http://www.cs.toronto.edu/~fritz/absps/ws.pdf)\n\n## Dependencies\n\n1. Install miniconda <http://conda.pydata.org/miniconda.html>\n2. Do `conda env create`\n3. Enter the env `source activate generative-models`\n4. Install [Tensorflow](https://www.tensorflow.org/get_started/os_setup)\n5. Install [Pytorch](https://github.com/pytorch/pytorch#installation)\n"
  },
  {
    "path": "VAE/adversarial_autoencoder/aae_pytorch.py",
    "content": "import torch\nimport torch.nn\nimport torch.nn.functional as nn\nimport torch.autograd as autograd\nimport torch.optim as optim\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 32\nz_dim = 5\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\ncnt = 0\nlr = 1e-3\n\n\n# Encoder\nQ = torch.nn.Sequential(\n    torch.nn.Linear(X_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, z_dim)\n)\n\n# Decoder\nP = torch.nn.Sequential(\n    torch.nn.Linear(z_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, X_dim),\n    torch.nn.Sigmoid()\n)\n\n# Discriminator\nD = torch.nn.Sequential(\n    torch.nn.Linear(z_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, 1),\n    torch.nn.Sigmoid()\n)\n\n\ndef reset_grad():\n    Q.zero_grad()\n    P.zero_grad()\n    D.zero_grad()\n\n\ndef sample_X(size, include_y=False):\n    X, y = mnist.train.next_batch(size)\n    X = Variable(torch.from_numpy(X))\n\n    if include_y:\n        y = np.argmax(y, axis=1).astype(np.int)\n        y = Variable(torch.from_numpy(y))\n        return X, y\n\n    return X\n\n\nQ_solver = optim.Adam(Q.parameters(), lr=lr)\nP_solver = optim.Adam(P.parameters(), lr=lr)\nD_solver = optim.Adam(D.parameters(), lr=lr)\n\n\nfor it in range(1000000):\n    X = sample_X(mb_size)\n\n    \"\"\" Reconstruction phase \"\"\"\n    z_sample = Q(X)\n    X_sample = P(z_sample)\n\n    recon_loss = nn.binary_cross_entropy(X_sample, X)\n\n    recon_loss.backward()\n    P_solver.step()\n    Q_solver.step()\n    reset_grad()\n\n    \"\"\" Regularization phase \"\"\"\n    # Discriminator\n    z_real = Variable(torch.randn(mb_size, z_dim))\n    z_fake = Q(X)\n\n    D_real = D(z_real)\n    D_fake = D(z_fake)\n\n    D_loss = -torch.mean(torch.log(D_real) + torch.log(1 - D_fake))\n\n    D_loss.backward()\n    D_solver.step()\n    reset_grad()\n\n    # Generator\n    z_fake = Q(X)\n    D_fake = D(z_fake)\n\n    G_loss = -torch.mean(torch.log(D_fake))\n\n    G_loss.backward()\n    Q_solver.step()\n    reset_grad()\n\n    # Print and plot every now and then\n    if it % 1000 == 0:\n        print('Iter-{}; D_loss: {:.4}; G_loss: {:.4}; recon_loss: {:.4}'\n              .format(it, D_loss.data[0], G_loss.data[0], recon_loss.data[0]))\n\n        samples = P(z_real).data.numpy()[:16]\n\n        fig = plt.figure(figsize=(4, 4))\n        gs = gridspec.GridSpec(4, 4)\n        gs.update(wspace=0.05, hspace=0.05)\n\n        for i, sample in enumerate(samples):\n            ax = plt.subplot(gs[i])\n            plt.axis('off')\n            ax.set_xticklabels([])\n            ax.set_yticklabels([])\n            ax.set_aspect('equal')\n            plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n        if not os.path.exists('out/'):\n            os.makedirs('out/')\n\n        plt.savefig('out/{}.png'\n                    .format(str(cnt).zfill(3)), bbox_inches='tight')\n        cnt += 1\n        plt.close(fig)\n"
  },
  {
    "path": "VAE/adversarial_autoencoder/aae_tensorflow.py",
    "content": "import tensorflow as tf\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 32\nz_dim = 10\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\nc = 0\nlr = 1e-3\n\n\ndef plot(samples):\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    return fig\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)\n    return tf.random_normal(shape=size, stddev=xavier_stddev)\n\n\n\"\"\" Q(z|X) \"\"\"\nX = tf.placeholder(tf.float32, shape=[None, X_dim])\nz = tf.placeholder(tf.float32, shape=[None, z_dim])\n\nQ_W1 = tf.Variable(xavier_init([X_dim, h_dim]))\nQ_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\n\nQ_W2 = tf.Variable(xavier_init([h_dim, z_dim]))\nQ_b2 = tf.Variable(tf.zeros(shape=[z_dim]))\n\ntheta_Q = [Q_W1, Q_W2, Q_b1, Q_b2]\n\n\ndef Q(X):\n    h = tf.nn.relu(tf.matmul(X, Q_W1) + Q_b1)\n    z = tf.matmul(h, Q_W2) + Q_b2\n    return z\n\n\n\"\"\" P(X|z) \"\"\"\nP_W1 = tf.Variable(xavier_init([z_dim, h_dim]))\nP_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\n\nP_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nP_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\ntheta_P = [P_W1, P_W2, P_b1, P_b2]\n\n\ndef P(z):\n    h = tf.nn.relu(tf.matmul(z, P_W1) + P_b1)\n    logits = tf.matmul(h, P_W2) + P_b2\n    prob = tf.nn.sigmoid(logits)\n    return prob, logits\n\n\n\"\"\" D(z) \"\"\"\nD_W1 = tf.Variable(xavier_init([z_dim, h_dim]))\nD_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\n\nD_W2 = tf.Variable(xavier_init([h_dim, 1]))\nD_b2 = tf.Variable(tf.zeros(shape=[1]))\n\ntheta_D = [D_W1, D_W2, D_b1, D_b2]\n\n\ndef D(z):\n    h = tf.nn.relu(tf.matmul(z, D_W1) + D_b1)\n    logits = tf.matmul(h, D_W2) + D_b2\n    prob = tf.nn.sigmoid(logits)\n    return prob\n\n\n\"\"\" Training \"\"\"\nz_sample = Q(X)\n_, logits = P(z_sample)\n\n# Sample from random z\nX_samples, _ = P(z)\n\n# E[log P(X|z)]\nrecon_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=X))\n\n# Adversarial loss to approx. Q(z|X)\nD_real = D(z)\nD_fake = D(z_sample)\n\nD_loss = -tf.reduce_mean(tf.log(D_real) + tf.log(1. - D_fake))\nG_loss = -tf.reduce_mean(tf.log(D_fake))\n\nAE_solver = tf.train.AdamOptimizer().minimize(recon_loss, var_list=theta_P + theta_Q)\nD_solver = tf.train.AdamOptimizer().minimize(D_loss, var_list=theta_D)\nG_solver = tf.train.AdamOptimizer().minimize(G_loss, var_list=theta_Q)\n\nsess = tf.Session()\nsess.run(tf.global_variables_initializer())\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\ni = 0\n\nfor it in range(1000000):\n    X_mb, _ = mnist.train.next_batch(mb_size)\n    z_mb = np.random.randn(mb_size, z_dim)\n\n    _, recon_loss_curr = sess.run([AE_solver, recon_loss], feed_dict={X: X_mb})\n    _, D_loss_curr = sess.run([D_solver, D_loss], feed_dict={X: X_mb, z: z_mb})\n    _, G_loss_curr = sess.run([G_solver, G_loss], feed_dict={X: X_mb})\n\n    if it % 1000 == 0:\n        print('Iter: {}; D_loss: {:.4}; G_loss: {:.4}; Recon_loss: {:.4}'\n              .format(it, D_loss_curr, G_loss_curr, recon_loss_curr))\n\n        samples = sess.run(X_samples, feed_dict={z: np.random.randn(16, z_dim)})\n\n        fig = plot(samples)\n        plt.savefig('out/{}.png'.format(str(i).zfill(3)), bbox_inches='tight')\n        i += 1\n        plt.close(fig)\n"
  },
  {
    "path": "VAE/adversarial_vb/avb_pytorch.py",
    "content": "import torch\nimport torch.nn\nimport torch.nn.functional as nn\nimport torch.autograd as autograd\nimport torch.optim as optim\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 32\nz_dim = 10\neps_dim = 4\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\ncnt = 0\nlr = 1e-3\n\n\ndef log(x):\n    return torch.log(x + 1e-8)\n\n\n# Encoder: q(z|x,eps)\nQ = torch.nn.Sequential(\n    torch.nn.Linear(X_dim + eps_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, z_dim)\n)\n\n# Decoder: p(x|z)\nP = torch.nn.Sequential(\n    torch.nn.Linear(z_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, X_dim),\n    torch.nn.Sigmoid()\n)\n\n# Discriminator: T(X, z)\nT = torch.nn.Sequential(\n    torch.nn.Linear(X_dim + z_dim, h_dim),\n    torch.nn.ReLU(),\n    torch.nn.Linear(h_dim, 1)\n)\n\n\ndef reset_grad():\n    Q.zero_grad()\n    P.zero_grad()\n    T.zero_grad()\n\n\ndef sample_X(size, include_y=False):\n    X, y = mnist.train.next_batch(size)\n    X = Variable(torch.from_numpy(X))\n\n    if include_y:\n        y = np.argmax(y, axis=1).astype(np.int)\n        y = Variable(torch.from_numpy(y))\n        return X, y\n\n    return X\n\n\nQ_solver = optim.Adam(Q.parameters(), lr=lr)\nP_solver = optim.Adam(P.parameters(), lr=lr)\nT_solver = optim.Adam(T.parameters(), lr=lr)\n\n\nfor it in range(1000000):\n    X = sample_X(mb_size)\n    eps = Variable(torch.randn(mb_size, eps_dim))\n    z = Variable(torch.randn(mb_size, z_dim))\n\n    # Optimize VAE\n    z_sample = Q(torch.cat([X, eps], 1))\n    X_sample = P(z_sample)\n    T_sample = T(torch.cat([X, z_sample], 1))\n\n    disc = torch.mean(-T_sample)\n    loglike = -nn.binary_cross_entropy(X_sample, X, size_average=False) / mb_size\n\n    elbo = -(disc + loglike)\n\n    elbo.backward()\n    Q_solver.step()\n    P_solver.step()\n    reset_grad()\n\n    # Discriminator T(X, z)\n    z_sample = Q(torch.cat([X, eps], 1))\n    T_q = nn.sigmoid(T(torch.cat([X, z_sample], 1)))\n    T_prior = nn.sigmoid(T(torch.cat([X, z], 1)))\n\n    T_loss = -torch.mean(log(T_q) + log(1. - T_prior))\n\n    T_loss.backward()\n    T_solver.step()\n    reset_grad()\n\n    # Print and plot every now and then\n    if it % 1000 == 0:\n        print('Iter-{}; ELBO: {:.4}; T_loss: {:.4}'\n              .format(it, -elbo.data[0], -T_loss.data[0]))\n\n        samples = P(z).data.numpy()[:16]\n\n        fig = plt.figure(figsize=(4, 4))\n        gs = gridspec.GridSpec(4, 4)\n        gs.update(wspace=0.05, hspace=0.05)\n\n        for i, sample in enumerate(samples):\n            ax = plt.subplot(gs[i])\n            plt.axis('off')\n            ax.set_xticklabels([])\n            ax.set_yticklabels([])\n            ax.set_aspect('equal')\n            plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n        if not os.path.exists('out/'):\n            os.makedirs('out/')\n\n        plt.savefig('out/{}.png'\n                    .format(str(cnt).zfill(3)), bbox_inches='tight')\n        cnt += 1\n        plt.close(fig)\n"
  },
  {
    "path": "VAE/adversarial_vb/avb_tensorflow.py",
    "content": "import tensorflow as tf\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 32\nz_dim = 10\neps_dim = 4\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\nc = 0\nlr = 1e-3\n\n\ndef log(x):\n    return tf.log(x + 1e-8)\n\n\ndef plot(samples):\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    return fig\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)\n    return tf.random_normal(shape=size, stddev=xavier_stddev)\n\n\n\"\"\" Q(z|X,eps) \"\"\"\nX = tf.placeholder(tf.float32, shape=[None, X_dim])\nz = tf.placeholder(tf.float32, shape=[None, z_dim])\neps = tf.placeholder(tf.float32, shape=[None, eps_dim])\n\nQ_W1 = tf.Variable(xavier_init([X_dim + eps_dim, h_dim]))\nQ_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nQ_W2 = tf.Variable(xavier_init([h_dim, z_dim]))\nQ_b2 = tf.Variable(tf.zeros(shape=[z_dim]))\n\ntheta_Q = [Q_W1, Q_W2, Q_b1, Q_b2]\n\n\ndef Q(X, eps):\n    inputs = tf.concat(axis=1, values=[X, eps])\n    h = tf.nn.relu(tf.matmul(inputs, Q_W1) + Q_b1)\n    z = tf.matmul(h, Q_W2) + Q_b2\n    return z\n\n\n\"\"\" P(X|z) \"\"\"\nP_W1 = tf.Variable(xavier_init([z_dim, h_dim]))\nP_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nP_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nP_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\ntheta_P = [P_W1, P_W2, P_b1, P_b2]\n\n\ndef P(z):\n    h = tf.nn.relu(tf.matmul(z, P_W1) + P_b1)\n    logits = tf.matmul(h, P_W2) + P_b2\n    prob = tf.nn.sigmoid(logits)\n    return prob, logits\n\n\n\"\"\" D(z) \"\"\"\nD_W1 = tf.Variable(xavier_init([X_dim + z_dim, h_dim]))\nD_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\nD_W2 = tf.Variable(xavier_init([h_dim, 1]))\nD_b2 = tf.Variable(tf.zeros(shape=[1]))\n\ntheta_D = [D_W1, D_W2, D_b1, D_b2]\n\n\ndef D(X, z):\n    inputs = tf.concat([X, z], axis=1)\n    h = tf.nn.relu(tf.matmul(inputs, D_W1) + D_b1)\n    return tf.matmul(h, D_W2) + D_b2\n\n\n\"\"\" Training \"\"\"\nz_sample = Q(X, eps)\n_, X_logits = P(z_sample)\nD_sample = D(X, z_sample)\n\nD_q = tf.nn.sigmoid(D(X, z_sample))\nD_prior = tf.nn.sigmoid(D(X, z))\n\n# Sample from random z\nX_samples, _ = P(z)\n\ndisc = tf.reduce_mean(-D_sample)\nnll = tf.reduce_sum(\n    tf.nn.sigmoid_cross_entropy_with_logits(logits=X_logits, labels=X),\n    axis=1\n)\nloglike = -tf.reduce_mean(nll)\n\nelbo = disc + loglike\nD_loss = tf.reduce_mean(log(D_q) + log(1. - D_prior))\n\nVAE_solver = tf.train.AdamOptimizer().minimize(-elbo, var_list=theta_P+theta_Q)\nD_solver = tf.train.AdamOptimizer().minimize(-D_loss, var_list=theta_D)\n\nsess = tf.Session()\nsess.run(tf.global_variables_initializer())\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\ni = 0\n\nfor it in range(1000000):\n    X_mb, _ = mnist.train.next_batch(mb_size)\n    eps_mb = np.random.randn(mb_size, eps_dim)\n    z_mb = np.random.randn(mb_size, z_dim)\n\n    _, elbo_curr = sess.run([VAE_solver, elbo],\n                            feed_dict={X: X_mb, eps: eps_mb, z: z_mb})\n\n    _, D_loss_curr = sess.run([D_solver, D_loss],\n                              feed_dict={X: X_mb, eps: eps_mb, z: z_mb})\n\n    if it % 1000 == 0:\n        print('Iter: {}; ELBO: {:.4}; D_Loss: {:.4}'\n              .format(it, elbo_curr, D_loss_curr))\n\n        samples = sess.run(X_samples, feed_dict={z: np.random.randn(16, z_dim)})\n\n        fig = plot(samples)\n        plt.savefig('out/{}.png'.format(str(i).zfill(3)), bbox_inches='tight')\n        i += 1\n        plt.close(fig)\n"
  },
  {
    "path": "VAE/conditional_vae/cvae_pytorch.py",
    "content": "import torch\nimport torch.nn.functional as nn\nimport torch.autograd as autograd\nimport torch.optim as optim\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 64\nZ_dim = 100\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\ncnt = 0\nlr = 1e-3\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / np.sqrt(in_dim / 2.)\n    return Variable(torch.randn(*size) * xavier_stddev, requires_grad=True)\n\n\n# =============================== Q(z|X) ======================================\n\nWxh = xavier_init(size=[X_dim + y_dim, h_dim])\nbxh = Variable(torch.zeros(h_dim), requires_grad=True)\n\nWhz_mu = xavier_init(size=[h_dim, Z_dim])\nbhz_mu = Variable(torch.zeros(Z_dim), requires_grad=True)\n\nWhz_var = xavier_init(size=[h_dim, Z_dim])\nbhz_var = Variable(torch.zeros(Z_dim), requires_grad=True)\n\n\ndef Q(X, c):\n    inputs = torch.cat([X, c], 1)\n    h = nn.relu(inputs @ Wxh + bxh.repeat(inputs.size(0), 1))\n    z_mu = h @ Whz_mu + bhz_mu.repeat(h.size(0), 1)\n    z_var = h @ Whz_var + bhz_var.repeat(h.size(0), 1)\n    return z_mu, z_var\n\n\ndef sample_z(mu, log_var):\n    eps = Variable(torch.randn(mb_size, Z_dim))\n    return mu + torch.exp(log_var / 2) * eps\n\n\n# =============================== P(X|z) ======================================\n\nWzh = xavier_init(size=[Z_dim + y_dim, h_dim])\nbzh = Variable(torch.zeros(h_dim), requires_grad=True)\n\nWhx = xavier_init(size=[h_dim, X_dim])\nbhx = Variable(torch.zeros(X_dim), requires_grad=True)\n\n\ndef P(z, c):\n    inputs = torch.cat([z, c], 1)\n    h = nn.relu(inputs @ Wzh + bzh.repeat(inputs.size(0), 1))\n    X = nn.sigmoid(h @ Whx + bhx.repeat(h.size(0), 1))\n    return X\n\n\n# =============================== TRAINING ====================================\n\nparams = [Wxh, bxh, Whz_mu, bhz_mu, Whz_var, bhz_var,\n          Wzh, bzh, Whx, bhx]\n\nsolver = optim.Adam(params, lr=lr)\n\nfor it in range(100000):\n    X, c = mnist.train.next_batch(mb_size)\n    X = Variable(torch.from_numpy(X))\n    c = Variable(torch.from_numpy(c.astype('float32')))\n\n    # Forward\n    z_mu, z_var = Q(X, c)\n    z = sample_z(z_mu, z_var)\n    X_sample = P(z, c)\n\n    # Loss\n    recon_loss = nn.binary_cross_entropy(X_sample, X, size_average=False) / mb_size\n    kl_loss = torch.mean(0.5 * torch.sum(torch.exp(z_var) + z_mu**2 - 1. - z_var, 1))\n    loss = recon_loss + kl_loss\n\n    # Backward\n    loss.backward()\n\n    # Update\n    solver.step()\n\n    # Housekeeping\n    for p in params:\n        if p.grad is not None:\n            data = p.grad.data\n            p.grad = Variable(data.new().resize_as_(data).zero_())\n\n    # Print and plot every now and then\n    if it % 1000 == 0:\n        print('Iter-{}; Loss: {:.4}'.format(it, loss.data[0]))\n\n        c = np.zeros(shape=[mb_size, y_dim], dtype='float32')\n        c[:, np.random.randint(0, 10)] = 1.\n        c = Variable(torch.from_numpy(c))\n        z = Variable(torch.randn(mb_size, Z_dim))\n        samples = P(z, c).data.numpy()[:16]\n\n        fig = plt.figure(figsize=(4, 4))\n        gs = gridspec.GridSpec(4, 4)\n        gs.update(wspace=0.05, hspace=0.05)\n\n        for i, sample in enumerate(samples):\n            ax = plt.subplot(gs[i])\n            plt.axis('off')\n            ax.set_xticklabels([])\n            ax.set_yticklabels([])\n            ax.set_aspect('equal')\n            plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n        if not os.path.exists('out/'):\n            os.makedirs('out/')\n\n        plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight')\n        cnt += 1\n        plt.close(fig)\n"
  },
  {
    "path": "VAE/conditional_vae/cvae_tensorflow.py",
    "content": "import tensorflow as tf\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 64\nz_dim = 100\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\nc = 0\nlr = 1e-3\n\n\ndef plot(samples):\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    return fig\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)\n    return tf.random_normal(shape=size, stddev=xavier_stddev)\n\n\n# =============================== Q(z|X) ======================================\n\nX = tf.placeholder(tf.float32, shape=[None, X_dim])\nc = tf.placeholder(tf.float32, shape=[None, y_dim])\nz = tf.placeholder(tf.float32, shape=[None, z_dim])\n\nQ_W1 = tf.Variable(xavier_init([X_dim + y_dim, h_dim]))\nQ_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\n\nQ_W2_mu = tf.Variable(xavier_init([h_dim, z_dim]))\nQ_b2_mu = tf.Variable(tf.zeros(shape=[z_dim]))\n\nQ_W2_sigma = tf.Variable(xavier_init([h_dim, z_dim]))\nQ_b2_sigma = tf.Variable(tf.zeros(shape=[z_dim]))\n\n\ndef Q(X, c):\n    inputs = tf.concat(axis=1, values=[X, c])\n    h = tf.nn.relu(tf.matmul(inputs, Q_W1) + Q_b1)\n    z_mu = tf.matmul(h, Q_W2_mu) + Q_b2_mu\n    z_logvar = tf.matmul(h, Q_W2_sigma) + Q_b2_sigma\n    return z_mu, z_logvar\n\n\ndef sample_z(mu, log_var):\n    eps = tf.random_normal(shape=tf.shape(mu))\n    return mu + tf.exp(log_var / 2) * eps\n\n\n# =============================== P(X|z) ======================================\n\nP_W1 = tf.Variable(xavier_init([z_dim + y_dim, h_dim]))\nP_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\n\nP_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nP_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\n\ndef P(z, c):\n    inputs = tf.concat(axis=1, values=[z, c])\n    h = tf.nn.relu(tf.matmul(inputs, P_W1) + P_b1)\n    logits = tf.matmul(h, P_W2) + P_b2\n    prob = tf.nn.sigmoid(logits)\n    return prob, logits\n\n\n# =============================== TRAINING ====================================\n\nz_mu, z_logvar = Q(X, c)\nz_sample = sample_z(z_mu, z_logvar)\n_, logits = P(z_sample, c)\n\n# Sampling from random z\nX_samples, _ = P(z, c)\n\n# E[log P(X|z)]\nrecon_loss = tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=X), 1)\n# D_KL(Q(z|X) || P(z|X)); calculate in closed form as both dist. are Gaussian\nkl_loss = 0.5 * tf.reduce_sum(tf.exp(z_logvar) + z_mu**2 - 1. - z_logvar, 1)\n# VAE loss\nvae_loss = tf.reduce_mean(recon_loss + kl_loss)\n\nsolver = tf.train.AdamOptimizer().minimize(vae_loss)\n\nsess = tf.Session()\nsess.run(tf.global_variables_initializer())\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\ni = 0\n\nfor it in range(1000000):\n    X_mb, y_mb = mnist.train.next_batch(mb_size)\n\n    _, loss = sess.run([solver, vae_loss], feed_dict={X: X_mb, c: y_mb})\n\n    if it % 1000 == 0:\n        print('Iter: {}'.format(it))\n        print('Loss: {:.4}'. format(loss))\n        print()\n\n        y = np.zeros(shape=[16, y_dim])\n        y[:, np.random.randint(0, y_dim)] = 1.\n\n        samples = sess.run(X_samples,\n                           feed_dict={z: np.random.randn(16, z_dim), c: y})\n\n        fig = plot(samples)\n        plt.savefig('out/{}.png'.format(str(i).zfill(3)), bbox_inches='tight')\n        i += 1\n        plt.close(fig)\n"
  },
  {
    "path": "VAE/denoising_vae/dvae_pytorch.py",
    "content": "import torch\nimport torch.nn.functional as nn\nimport torch.autograd as autograd\nimport torch.optim as optim\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 64\nZ_dim = 100\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\nc = 0\nlr = 1e-3\nnoise_factor = .25\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / np.sqrt(in_dim / 2.)\n    return Variable(torch.randn(*size) * xavier_stddev, requires_grad=True)\n\n\n\"\"\" Q(z|X) \"\"\"\nWxh = xavier_init(size=[X_dim, h_dim])\nbxh = Variable(torch.zeros(h_dim), requires_grad=True)\n\nWhz_mu = xavier_init(size=[h_dim, Z_dim])\nbhz_mu = Variable(torch.zeros(Z_dim), requires_grad=True)\n\nWhz_var = xavier_init(size=[h_dim, Z_dim])\nbhz_var = Variable(torch.zeros(Z_dim), requires_grad=True)\n\n\ndef Q(X):\n    h = nn.relu(X @ Wxh + bxh.repeat(X.size(0), 1))\n    z_mu = h @ Whz_mu + bhz_mu.repeat(h.size(0), 1)\n    z_var = h @ Whz_var + bhz_var.repeat(h.size(0), 1)\n    return z_mu, z_var\n\n\ndef sample_z(mu, log_var):\n    eps = Variable(torch.randn(mb_size, Z_dim))\n    return mu + torch.exp(log_var / 2) * eps\n\n\n\"\"\" P(X|z) \"\"\"\nWzh = xavier_init(size=[Z_dim, h_dim])\nbzh = Variable(torch.zeros(h_dim), requires_grad=True)\n\nWhx = xavier_init(size=[h_dim, X_dim])\nbhx = Variable(torch.zeros(X_dim), requires_grad=True)\n\n\ndef P(z):\n    h = nn.relu(z @ Wzh + bzh.repeat(z.size(0), 1))\n    X = nn.sigmoid(h @ Whx + bhx.repeat(h.size(0), 1))\n    return X\n\n\n\"\"\" Training \"\"\"\nparams = [Wxh, bxh, Whz_mu, bhz_mu, Whz_var, bhz_var,\n          Wzh, bzh, Whx, bhx]\n\nsolver = optim.Adam(params, lr=lr)\n\nfor it in range(100000):\n    X, _ = mnist.train.next_batch(mb_size)\n    X = Variable(torch.from_numpy(X))\n\n    # Add noise\n    X_noise = X + noise_factor * Variable(torch.randn(X.size()))\n    X_noise.data.clamp_(0., 1.)\n\n    # Forward\n    z_mu, z_var = Q(X_noise)\n    z = sample_z(z_mu, z_var)\n    X_sample = P(z)\n\n    torch.nn.BCELoss\n    recon_loss = nn.binary_cross_entropy(X_sample, X, size_average=False) / mb_size\n    kl_loss = torch.mean(0.5 * torch.sum(torch.exp(z_var) + z_mu**2 - 1. - z_var, 1))\n    loss = recon_loss + kl_loss\n\n    # Backward\n    loss.backward()\n\n    # Update\n    solver.step()\n\n    # Housekeeping\n    for p in params:\n        if p.grad is not None:\n            data = p.grad.data\n            p.grad = Variable(data.new().resize_as_(data).zero_())\n\n    # Print and plot every now and then\n    if it % 1000 == 0:\n        print('Iter-{}; Loss: {:.4}'.format(it, loss.data[0]))\n\n        z = Variable(torch.randn(mb_size, Z_dim))\n        samples = P(z).data.numpy()[:16]\n\n        fig = plt.figure(figsize=(4, 4))\n        gs = gridspec.GridSpec(4, 4)\n        gs.update(wspace=0.05, hspace=0.05)\n\n        for i, sample in enumerate(samples):\n            ax = plt.subplot(gs[i])\n            plt.axis('off')\n            ax.set_xticklabels([])\n            ax.set_yticklabels([])\n            ax.set_aspect('equal')\n            plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n        if not os.path.exists('out/'):\n            os.makedirs('out/')\n\n        plt.savefig('out/{}.png'.format(str(c).zfill(3)), bbox_inches='tight')\n        c += 1\n        plt.close(fig)\n"
  },
  {
    "path": "VAE/denoising_vae/dvae_tensorflow.py",
    "content": "import tensorflow as tf\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 64\nz_dim = 100\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\nc = 0\nlr = 1e-3\nnoise_factor = 0.25\n\n\ndef plot(samples):\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    return fig\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)\n    return tf.random_normal(shape=size, stddev=xavier_stddev)\n\n\n\"\"\" Q(X|z) \"\"\"\nX = tf.placeholder(tf.float32, shape=[None, X_dim])\nz = tf.placeholder(tf.float32, shape=[None, z_dim])\n\nQ_W1 = tf.Variable(xavier_init([X_dim, h_dim]))\nQ_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\n\nQ_W2_mu = tf.Variable(xavier_init([h_dim, z_dim]))\nQ_b2_mu = tf.Variable(tf.zeros(shape=[z_dim]))\n\nQ_W2_sigma = tf.Variable(xavier_init([h_dim, z_dim]))\nQ_b2_sigma = tf.Variable(tf.zeros(shape=[z_dim]))\n\n\ndef Q(X):\n    h = tf.nn.relu(tf.matmul(X, Q_W1) + Q_b1)\n    z_mu = tf.matmul(h, Q_W2_mu) + Q_b2_mu\n    z_logvar = tf.matmul(h, Q_W2_sigma) + Q_b2_sigma\n    return z_mu, z_logvar\n\n\ndef sample_z(mu, log_var):\n    eps = tf.random_normal(shape=tf.shape(mu))\n    return mu + tf.exp(log_var / 2) * eps\n\n\n\"\"\" P(X|z) \"\"\"\nP_W1 = tf.Variable(xavier_init([z_dim, h_dim]))\nP_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\n\nP_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nP_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\n\ndef P(z):\n    h = tf.nn.relu(tf.matmul(z, P_W1) + P_b1)\n    logits = tf.matmul(h, P_W2) + P_b2\n    prob = tf.nn.sigmoid(logits)\n    return prob, logits\n\n\n\"\"\" Training \"\"\"\n# Add noise to X\nX_noise = X + noise_factor * tf.random_normal(tf.shape(X))\nX_noise = tf.clip_by_value(X_noise, 0., 1.)\n\nz_mu, z_logvar = Q(X_noise)\nz_sample = sample_z(z_mu, z_logvar)\n_, logits = P(z_sample)\n\n# Sample from random z\nX_samples, _ = P(z)\n\n# E[log P(X|z)]\nrecon_loss = tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=X), 1)\n# D_KL(Q(z|X_noise) || P(z|X)); calculate in closed form as both dist. are Gaussian\nkl_loss = 0.5 * tf.reduce_sum(tf.exp(z_logvar) + z_mu**2 - 1. - z_logvar, 1)\n# VAE loss\nvae_loss = tf.reduce_mean(recon_loss + kl_loss)\n\nsolver = tf.train.AdamOptimizer().minimize(vae_loss)\n\nsess = tf.Session()\nsess.run(tf.global_variables_initializer())\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\ni = 0\n\nfor it in range(1000000):\n    X_mb, _ = mnist.train.next_batch(mb_size)\n\n    _, loss = sess.run([solver, vae_loss], feed_dict={X: X_mb})\n\n    if it % 1000 == 0:\n        print('Iter: {}; Loss: {:.4}'.format(it, loss))\n\n        samples = sess.run(X_samples, feed_dict={z: np.random.randn(16, z_dim)})\n\n        fig = plot(samples)\n        plt.savefig('out/{}.png'.format(str(i).zfill(3)), bbox_inches='tight')\n        i += 1\n        plt.close(fig)\n"
  },
  {
    "path": "VAE/vanilla_vae/vae_pytorch.py",
    "content": "import torch\nimport torch.nn.functional as nn\nimport torch.autograd as autograd\nimport torch.optim as optim\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom torch.autograd import Variable\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 64\nZ_dim = 100\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\nc = 0\nlr = 1e-3\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / np.sqrt(in_dim / 2.)\n    return Variable(torch.randn(*size) * xavier_stddev, requires_grad=True)\n\n\n# =============================== Q(z|X) ======================================\n\nWxh = xavier_init(size=[X_dim, h_dim])\nbxh = Variable(torch.zeros(h_dim), requires_grad=True)\n\nWhz_mu = xavier_init(size=[h_dim, Z_dim])\nbhz_mu = Variable(torch.zeros(Z_dim), requires_grad=True)\n\nWhz_var = xavier_init(size=[h_dim, Z_dim])\nbhz_var = Variable(torch.zeros(Z_dim), requires_grad=True)\n\n\ndef Q(X):\n    h = nn.relu(X @ Wxh + bxh.repeat(X.size(0), 1))\n    z_mu = h @ Whz_mu + bhz_mu.repeat(h.size(0), 1)\n    z_var = h @ Whz_var + bhz_var.repeat(h.size(0), 1)\n    return z_mu, z_var\n\n\ndef sample_z(mu, log_var):\n    eps = Variable(torch.randn(mb_size, Z_dim))\n    return mu + torch.exp(log_var / 2) * eps\n\n\n# =============================== P(X|z) ======================================\n\nWzh = xavier_init(size=[Z_dim, h_dim])\nbzh = Variable(torch.zeros(h_dim), requires_grad=True)\n\nWhx = xavier_init(size=[h_dim, X_dim])\nbhx = Variable(torch.zeros(X_dim), requires_grad=True)\n\n\ndef P(z):\n    h = nn.relu(z @ Wzh + bzh.repeat(z.size(0), 1))\n    X = nn.sigmoid(h @ Whx + bhx.repeat(h.size(0), 1))\n    return X\n\n\n# =============================== TRAINING ====================================\n\nparams = [Wxh, bxh, Whz_mu, bhz_mu, Whz_var, bhz_var,\n          Wzh, bzh, Whx, bhx]\n\nsolver = optim.Adam(params, lr=lr)\n\nfor it in range(100000):\n    X, _ = mnist.train.next_batch(mb_size)\n    X = Variable(torch.from_numpy(X))\n\n    # Forward\n    z_mu, z_var = Q(X)\n    z = sample_z(z_mu, z_var)\n    X_sample = P(z)\n\n    # Loss\n    recon_loss = nn.binary_cross_entropy(X_sample, X, size_average=False) / mb_size\n    kl_loss = torch.mean(0.5 * torch.sum(torch.exp(z_var) + z_mu**2 - 1. - z_var, 1))\n    loss = recon_loss + kl_loss\n\n    # Backward\n    loss.backward()\n\n    # Update\n    solver.step()\n\n    # Housekeeping\n    for p in params:\n        if p.grad is not None:\n            data = p.grad.data\n            p.grad = Variable(data.new().resize_as_(data).zero_())\n\n    # Print and plot every now and then\n    if it % 1000 == 0:\n        print('Iter-{}; Loss: {:.4}'.format(it, loss.data[0]))\n\n        samples = P(z).data.numpy()[:16]\n\n        fig = plt.figure(figsize=(4, 4))\n        gs = gridspec.GridSpec(4, 4)\n        gs.update(wspace=0.05, hspace=0.05)\n\n        for i, sample in enumerate(samples):\n            ax = plt.subplot(gs[i])\n            plt.axis('off')\n            ax.set_xticklabels([])\n            ax.set_yticklabels([])\n            ax.set_aspect('equal')\n            plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n        if not os.path.exists('out/'):\n            os.makedirs('out/')\n\n        plt.savefig('out/{}.png'.format(str(c).zfill(3)), bbox_inches='tight')\n        c += 1\n        plt.close(fig)\n"
  },
  {
    "path": "VAE/vanilla_vae/vae_tensorflow.py",
    "content": "import tensorflow as tf\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport os\nfrom tensorflow.examples.tutorials.mnist import input_data\n\n\nmnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)\nmb_size = 64\nz_dim = 100\nX_dim = mnist.train.images.shape[1]\ny_dim = mnist.train.labels.shape[1]\nh_dim = 128\nc = 0\nlr = 1e-3\n\n\ndef plot(samples):\n    fig = plt.figure(figsize=(4, 4))\n    gs = gridspec.GridSpec(4, 4)\n    gs.update(wspace=0.05, hspace=0.05)\n\n    for i, sample in enumerate(samples):\n        ax = plt.subplot(gs[i])\n        plt.axis('off')\n        ax.set_xticklabels([])\n        ax.set_yticklabels([])\n        ax.set_aspect('equal')\n        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n\n    return fig\n\n\ndef xavier_init(size):\n    in_dim = size[0]\n    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)\n    return tf.random_normal(shape=size, stddev=xavier_stddev)\n\n\n# =============================== Q(z|X) ======================================\n\nX = tf.placeholder(tf.float32, shape=[None, X_dim])\nz = tf.placeholder(tf.float32, shape=[None, z_dim])\n\nQ_W1 = tf.Variable(xavier_init([X_dim, h_dim]))\nQ_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\n\nQ_W2_mu = tf.Variable(xavier_init([h_dim, z_dim]))\nQ_b2_mu = tf.Variable(tf.zeros(shape=[z_dim]))\n\nQ_W2_sigma = tf.Variable(xavier_init([h_dim, z_dim]))\nQ_b2_sigma = tf.Variable(tf.zeros(shape=[z_dim]))\n\n\ndef Q(X):\n    h = tf.nn.relu(tf.matmul(X, Q_W1) + Q_b1)\n    z_mu = tf.matmul(h, Q_W2_mu) + Q_b2_mu\n    z_logvar = tf.matmul(h, Q_W2_sigma) + Q_b2_sigma\n    return z_mu, z_logvar\n\n\ndef sample_z(mu, log_var):\n    eps = tf.random_normal(shape=tf.shape(mu))\n    return mu + tf.exp(log_var / 2) * eps\n\n\n# =============================== P(X|z) ======================================\n\nP_W1 = tf.Variable(xavier_init([z_dim, h_dim]))\nP_b1 = tf.Variable(tf.zeros(shape=[h_dim]))\n\nP_W2 = tf.Variable(xavier_init([h_dim, X_dim]))\nP_b2 = tf.Variable(tf.zeros(shape=[X_dim]))\n\n\ndef P(z):\n    h = tf.nn.relu(tf.matmul(z, P_W1) + P_b1)\n    logits = tf.matmul(h, P_W2) + P_b2\n    prob = tf.nn.sigmoid(logits)\n    return prob, logits\n\n\n# =============================== TRAINING ====================================\n\nz_mu, z_logvar = Q(X)\nz_sample = sample_z(z_mu, z_logvar)\n_, logits = P(z_sample)\n\n# Sampling from random z\nX_samples, _ = P(z)\n\n# E[log P(X|z)]\nrecon_loss = tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=X), 1)\n# D_KL(Q(z|X) || P(z)); calculate in closed form as both dist. are Gaussian\nkl_loss = 0.5 * tf.reduce_sum(tf.exp(z_logvar) + z_mu**2 - 1. - z_logvar, 1)\n# VAE loss\nvae_loss = tf.reduce_mean(recon_loss + kl_loss)\n\nsolver = tf.train.AdamOptimizer().minimize(vae_loss)\n\nsess = tf.Session()\nsess.run(tf.global_variables_initializer())\n\nif not os.path.exists('out/'):\n    os.makedirs('out/')\n\ni = 0\n\nfor it in range(1000000):\n    X_mb, _ = mnist.train.next_batch(mb_size)\n\n    _, loss = sess.run([solver, vae_loss], feed_dict={X: X_mb})\n\n    if it % 1000 == 0:\n        print('Iter: {}'.format(it))\n        print('Loss: {:.4}'. format(loss))\n        print()\n\n        samples = sess.run(X_samples, feed_dict={z: np.random.randn(16, z_dim)})\n\n        fig = plot(samples)\n        plt.savefig('out/{}.png'.format(str(i).zfill(3)), bbox_inches='tight')\n        i += 1\n        plt.close(fig)\n"
  },
  {
    "path": "environment.yml",
    "content": "name: generative-models\ndependencies:\n- python=3.5.1\n- numpy=1.11.0\n- scikit-learn=0.17.1\n- scipy=0.17.1\n- matplotlib=1.5.3\n- pip:\n    - keras==1.1.1\n"
  }
]