Repository: eriklindernoren/Keras-GAN Branch: master Commit: 3ff3be4b4b2f Files: 76 Total size: 179.3 KB Directory structure: gitextract_yxj1175v/ ├── .gitignore ├── LICENSE ├── README.md ├── aae/ │ ├── aae.py │ ├── images/ │ │ └── .gitignore │ └── saved_model/ │ └── .gitignore ├── acgan/ │ ├── acgan.py │ ├── images/ │ │ └── .gitignore │ └── saved_model/ │ └── .gitignore ├── bgan/ │ ├── bgan.py │ ├── images/ │ │ └── .gitignore │ └── saved_model/ │ └── .gitignore ├── bigan/ │ ├── bigan.py │ ├── images/ │ │ └── .gitignore │ └── saved_model/ │ └── .gitignore ├── ccgan/ │ ├── ccgan.py │ ├── images/ │ │ └── .gitignore │ └── saved_model/ │ └── .gitignore ├── cgan/ │ ├── cgan.py │ ├── images/ │ │ └── .gitignore │ └── saved_model/ │ └── .gitignore ├── cogan/ │ ├── cogan.py │ ├── images/ │ │ └── .gitignore │ └── saved_model/ │ └── .gitignore ├── context_encoder/ │ ├── context_encoder.py │ ├── images/ │ │ └── .gitignore │ └── saved_model/ │ └── .gitignore ├── cyclegan/ │ ├── cyclegan.py │ ├── data_loader.py │ ├── download_dataset.sh │ ├── images/ │ │ └── .gitignore │ └── saved_model/ │ └── .gitignore ├── dcgan/ │ ├── dcgan.py │ ├── images/ │ │ └── .gitignore │ └── saved_model/ │ └── .gitignore ├── discogan/ │ ├── data_loader.py │ ├── discogan.py │ ├── download_dataset.sh │ ├── images/ │ │ └── .gitignore │ └── saved_model/ │ └── .gitignore ├── dualgan/ │ ├── dualgan.py │ ├── images/ │ │ └── .gitignore │ └── saved_model/ │ └── .gitignore ├── gan/ │ ├── gan.py │ ├── images/ │ │ └── .gitignore │ └── saved_model/ │ └── .gitignore ├── infogan/ │ ├── images/ │ │ └── .gitignore │ ├── infogan.py │ └── saved_model/ │ └── .gitignore ├── lsgan/ │ ├── images/ │ │ └── .gitignore │ ├── lsgan.py │ └── saved_model/ │ └── .gitignore ├── pix2pix/ │ ├── data_loader.py │ ├── download_dataset.sh │ ├── images/ │ │ └── .gitignore │ ├── pix2pix.py │ └── saved_model/ │ └── .gitignore ├── pixelda/ │ ├── data_loader.py │ ├── images/ │ │ └── .gitignore │ ├── pixelda.py │ ├── saved_model/ │ │ └── .gitignore │ └── test.py ├── requirements.txt ├── sgan/ │ ├── images/ │ │ └── .gitignore │ ├── saved_model/ │ │ └── .gitignore │ └── sgan.py ├── srgan/ │ ├── data_loader.py │ ├── images/ │ │ └── .gitignore │ ├── saved_model/ │ │ └── .gitignore │ └── srgan.py ├── wgan/ │ ├── images/ │ │ └── .gitignore │ ├── saved_model/ │ │ └── .gitignore │ └── wgan.py └── wgan_gp/ ├── images/ │ └── .gitignore ├── saved_model/ │ └── .gitignore └── wgan_gp.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ */images/*.png */images/*.jpg */*.jpg */*.png *.json *.h5 *.hdf5 .DS_Store */datasets __pycache__ ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2017 Erik Linder-Norén Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================

**This repository has gone stale as I unfortunately do not have the time to maintain it anymore. If you would like to continue the development of it as a collaborator send me an email at eriklindernoren@gmail.com.** ## Keras-GAN Collection of Keras implementations of Generative Adversarial Networks (GANs) suggested in research papers. These models are in some cases simplified versions of the ones ultimately described in the papers, but I have chosen to focus on getting the core ideas covered instead of getting every layer configuration right. Contributions and suggestions of GAN varieties to implement are very welcomed. See also: [PyTorch-GAN](https://github.com/eriklindernoren/PyTorch-GAN) ## Table of Contents * [Installation](#installation) * [Implementations](#implementations) + [Auxiliary Classifier GAN](#ac-gan) + [Adversarial Autoencoder](#adversarial-autoencoder) + [Bidirectional GAN](#bigan) + [Boundary-Seeking GAN](#bgan) + [Conditional GAN](#cgan) + [Context-Conditional GAN](#cc-gan) + [Context Encoder](#context-encoder) + [Coupled GANs](#cogan) + [CycleGAN](#cyclegan) + [Deep Convolutional GAN](#dcgan) + [DiscoGAN](#discogan) + [DualGAN](#dualgan) + [Generative Adversarial Network](#gan) + [InfoGAN](#infogan) + [LSGAN](#lsgan) + [Pix2Pix](#pix2pix) + [PixelDA](#pixelda) + [Semi-Supervised GAN](#sgan) + [Super-Resolution GAN](#srgan) + [Wasserstein GAN](#wgan) + [Wasserstein GAN GP](#wgan-gp) ## Installation $ git clone https://github.com/eriklindernoren/Keras-GAN $ cd Keras-GAN/ $ sudo pip3 install -r requirements.txt ## Implementations ### AC-GAN Implementation of _Auxiliary Classifier Generative Adversarial Network_. [Code](acgan/acgan.py) Paper: https://arxiv.org/abs/1610.09585 #### Example ``` $ cd acgan/ $ python3 acgan.py ```

### Adversarial Autoencoder Implementation of _Adversarial Autoencoder_. [Code](aae/aae.py) Paper: https://arxiv.org/abs/1511.05644 #### Example ``` $ cd aae/ $ python3 aae.py ```

### BiGAN Implementation of _Bidirectional Generative Adversarial Network_. [Code](bigan/bigan.py) Paper: https://arxiv.org/abs/1605.09782 #### Example ``` $ cd bigan/ $ python3 bigan.py ``` ### BGAN Implementation of _Boundary-Seeking Generative Adversarial Networks_. [Code](bgan/bgan.py) Paper: https://arxiv.org/abs/1702.08431 #### Example ``` $ cd bgan/ $ python3 bgan.py ``` ### CC-GAN Implementation of _Semi-Supervised Learning with Context-Conditional Generative Adversarial Networks_. [Code](ccgan/ccgan.py) Paper: https://arxiv.org/abs/1611.06430 #### Example ``` $ cd ccgan/ $ python3 ccgan.py ```

### CGAN Implementation of _Conditional Generative Adversarial Nets_. [Code](cgan/cgan.py) Paper:https://arxiv.org/abs/1411.1784 #### Example ``` $ cd cgan/ $ python3 cgan.py ```

### Context Encoder Implementation of _Context Encoders: Feature Learning by Inpainting_. [Code](context_encoder/context_encoder.py) Paper: https://arxiv.org/abs/1604.07379 #### Example ``` $ cd context_encoder/ $ python3 context_encoder.py ```

### CoGAN Implementation of _Coupled generative adversarial networks_. [Code](cogan/cogan.py) Paper: https://arxiv.org/abs/1606.07536 #### Example ``` $ cd cogan/ $ python3 cogan.py ``` ### CycleGAN Implementation of _Unpaired Image-to-Image Translation using Cycle-Consistent Adversarial Networks_. [Code](cyclegan/cyclegan.py) Paper: https://arxiv.org/abs/1703.10593

#### Example ``` $ cd cyclegan/ $ bash download_dataset.sh apple2orange $ python3 cyclegan.py ```

### DCGAN Implementation of _Deep Convolutional Generative Adversarial Network_. [Code](dcgan/dcgan.py) Paper: https://arxiv.org/abs/1511.06434 #### Example ``` $ cd dcgan/ $ python3 dcgan.py ```

### DiscoGAN Implementation of _Learning to Discover Cross-Domain Relations with Generative Adversarial Networks_. [Code](discogan/discogan.py) Paper: https://arxiv.org/abs/1703.05192

#### Example ``` $ cd discogan/ $ bash download_dataset.sh edges2shoes $ python3 discogan.py ```

### DualGAN Implementation of _DualGAN: Unsupervised Dual Learning for Image-to-Image Translation_. [Code](dualgan/dualgan.py) Paper: https://arxiv.org/abs/1704.02510 #### Example ``` $ cd dualgan/ $ python3 dualgan.py ``` ### GAN Implementation of _Generative Adversarial Network_ with a MLP generator and discriminator. [Code](gan/gan.py) Paper: https://arxiv.org/abs/1406.2661 #### Example ``` $ cd gan/ $ python3 gan.py ```

### InfoGAN Implementation of _InfoGAN: Interpretable Representation Learning by Information Maximizing Generative Adversarial Nets_. [Code](infogan/infogan.py) Paper: https://arxiv.org/abs/1606.03657 #### Example ``` $ cd infogan/ $ python3 infogan.py ```

### LSGAN Implementation of _Least Squares Generative Adversarial Networks_. [Code](lsgan/lsgan.py) Paper: https://arxiv.org/abs/1611.04076 #### Example ``` $ cd lsgan/ $ python3 lsgan.py ``` ### Pix2Pix Implementation of _Image-to-Image Translation with Conditional Adversarial Networks_. [Code](pix2pix/pix2pix.py) Paper: https://arxiv.org/abs/1611.07004

#### Example ``` $ cd pix2pix/ $ bash download_dataset.sh facades $ python3 pix2pix.py ```

### PixelDA Implementation of _Unsupervised Pixel-Level Domain Adaptation with Generative Adversarial Networks_. [Code](pixelda/pixelda.py) Paper: https://arxiv.org/abs/1612.05424 #### MNIST to MNIST-M Classification Trains a classifier on MNIST images that are translated to resemble MNIST-M (by performing unsupervised image-to-image domain adaptation). This model is compared to the naive solution of training a classifier on MNIST and evaluating it on MNIST-M. The naive model manages a 55% classification accuracy on MNIST-M while the one trained during domain adaptation gets a 95% classification accuracy. ``` $ cd pixelda/ $ python3 pixelda.py ``` | Method | Accuracy | | ------------ |:---------:| | Naive | 55% | | PixelDA | 95% | ### SGAN Implementation of _Semi-Supervised Generative Adversarial Network_. [Code](sgan/sgan.py) Paper: https://arxiv.org/abs/1606.01583 #### Example ``` $ cd sgan/ $ python3 sgan.py ```

### SRGAN Implementation of _Photo-Realistic Single Image Super-Resolution Using a Generative Adversarial Network_. [Code](srgan/srgan.py) Paper: https://arxiv.org/abs/1609.04802

#### Example ``` $ cd srgan/ $ python3 srgan.py ```

### WGAN Implementation of _Wasserstein GAN_ (with DCGAN generator and discriminator). [Code](wgan/wgan.py) Paper: https://arxiv.org/abs/1701.07875 #### Example ``` $ cd wgan/ $ python3 wgan.py ```

### WGAN GP Implementation of _Improved Training of Wasserstein GANs_. [Code](wgan_gp/wgan_gp.py) Paper: https://arxiv.org/abs/1704.00028 #### Example ``` $ cd wgan_gp/ $ python3 wgan_gp.py ```

================================================ FILE: aae/aae.py ================================================ from __future__ import print_function, division from keras.datasets import mnist from keras.layers import Input, Dense, Reshape, Flatten, Dropout, multiply, GaussianNoise from keras.layers import BatchNormalization, Activation, Embedding, ZeroPadding2D from keras.layers import MaxPooling2D, merge from keras.layers.advanced_activations import LeakyReLU from keras.layers.convolutional import UpSampling2D, Conv2D from keras.models import Sequential, Model from keras.optimizers import Adam from keras import losses from keras.utils import to_categorical import keras.backend as K import matplotlib.pyplot as plt import numpy as np class AdversarialAutoencoder(): def __init__(self): self.img_rows = 28 self.img_cols = 28 self.channels = 1 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.latent_dim = 10 optimizer = Adam(0.0002, 0.5) # Build and compile the discriminator self.discriminator = self.build_discriminator() self.discriminator.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) # Build the encoder / decoder self.encoder = self.build_encoder() self.decoder = self.build_decoder() img = Input(shape=self.img_shape) # The generator takes the image, encodes it and reconstructs it # from the encoding encoded_repr = self.encoder(img) reconstructed_img = self.decoder(encoded_repr) # For the adversarial_autoencoder model we will only train the generator self.discriminator.trainable = False # The discriminator determines validity of the encoding validity = self.discriminator(encoded_repr) # The adversarial_autoencoder model (stacked generator and discriminator) self.adversarial_autoencoder = Model(img, [reconstructed_img, validity]) self.adversarial_autoencoder.compile(loss=['mse', 'binary_crossentropy'], loss_weights=[0.999, 0.001], optimizer=optimizer) def build_encoder(self): # Encoder img = Input(shape=self.img_shape) h = Flatten()(img) h = Dense(512)(h) h = LeakyReLU(alpha=0.2)(h) h = Dense(512)(h) h = LeakyReLU(alpha=0.2)(h) mu = Dense(self.latent_dim)(h) log_var = Dense(self.latent_dim)(h) latent_repr = merge([mu, log_var], mode=lambda p: p[0] + K.random_normal(K.shape(p[0])) * K.exp(p[1] / 2), output_shape=lambda p: p[0]) return Model(img, latent_repr) def build_decoder(self): model = Sequential() model.add(Dense(512, input_dim=self.latent_dim)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(np.prod(self.img_shape), activation='tanh')) model.add(Reshape(self.img_shape)) model.summary() z = Input(shape=(self.latent_dim,)) img = model(z) return Model(z, img) def build_discriminator(self): model = Sequential() model.add(Dense(512, input_dim=self.latent_dim)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(256)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(1, activation="sigmoid")) model.summary() encoded_repr = Input(shape=(self.latent_dim, )) validity = model(encoded_repr) return Model(encoded_repr, validity) def train(self, epochs, batch_size=128, sample_interval=50): # Load the dataset (X_train, _), (_, _) = mnist.load_data() # Rescale -1 to 1 X_train = (X_train.astype(np.float32) - 127.5) / 127.5 X_train = np.expand_dims(X_train, axis=3) # Adversarial ground truths valid = np.ones((batch_size, 1)) fake = np.zeros((batch_size, 1)) for epoch in range(epochs): # --------------------- # Train Discriminator # --------------------- # Select a random batch of images idx = np.random.randint(0, X_train.shape[0], batch_size) imgs = X_train[idx] latent_fake = self.encoder.predict(imgs) latent_real = np.random.normal(size=(batch_size, self.latent_dim)) # Train the discriminator d_loss_real = self.discriminator.train_on_batch(latent_real, valid) d_loss_fake = self.discriminator.train_on_batch(latent_fake, fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # --------------------- # Train Generator # --------------------- # Train the generator g_loss = self.adversarial_autoencoder.train_on_batch(imgs, [imgs, valid]) # Plot the progress print ("%d [D loss: %f, acc: %.2f%%] [G loss: %f, mse: %f]" % (epoch, d_loss[0], 100*d_loss[1], g_loss[0], g_loss[1])) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.sample_images(epoch) def sample_images(self, epoch): r, c = 5, 5 z = np.random.normal(size=(r*c, self.latent_dim)) gen_imgs = self.decoder.predict(z) gen_imgs = 0.5 * gen_imgs + 0.5 fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): axs[i,j].imshow(gen_imgs[cnt, :,:,0], cmap='gray') axs[i,j].axis('off') cnt += 1 fig.savefig("images/mnist_%d.png" % epoch) plt.close() def save_model(self): def save(model, model_name): model_path = "saved_model/%s.json" % model_name weights_path = "saved_model/%s_weights.hdf5" % model_name options = {"file_arch": model_path, "file_weight": weights_path} json_string = model.to_json() open(options['file_arch'], 'w').write(json_string) model.save_weights(options['file_weight']) save(self.generator, "aae_generator") save(self.discriminator, "aae_discriminator") if __name__ == '__main__': aae = AdversarialAutoencoder() aae.train(epochs=20000, batch_size=32, sample_interval=200) ================================================ FILE: aae/images/.gitignore ================================================ * !.gitignore ================================================ FILE: aae/saved_model/.gitignore ================================================ * !.gitignore ================================================ FILE: acgan/acgan.py ================================================ from __future__ import print_function, division from keras.datasets import mnist from keras.layers import Input, Dense, Reshape, Flatten, Dropout, multiply from keras.layers import BatchNormalization, Activation, Embedding, ZeroPadding2D from keras.layers.advanced_activations import LeakyReLU from keras.layers.convolutional import UpSampling2D, Conv2D from keras.models import Sequential, Model from keras.optimizers import Adam import matplotlib.pyplot as plt import numpy as np class ACGAN(): def __init__(self): # Input shape self.img_rows = 28 self.img_cols = 28 self.channels = 1 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.num_classes = 10 self.latent_dim = 100 optimizer = Adam(0.0002, 0.5) losses = ['binary_crossentropy', 'sparse_categorical_crossentropy'] # Build and compile the discriminator self.discriminator = self.build_discriminator() self.discriminator.compile(loss=losses, optimizer=optimizer, metrics=['accuracy']) # Build the generator self.generator = self.build_generator() # The generator takes noise and the target label as input # and generates the corresponding digit of that label noise = Input(shape=(self.latent_dim,)) label = Input(shape=(1,)) img = self.generator([noise, label]) # For the combined model we will only train the generator self.discriminator.trainable = False # The discriminator takes generated image as input and determines validity # and the label of that image valid, target_label = self.discriminator(img) # The combined model (stacked generator and discriminator) # Trains the generator to fool the discriminator self.combined = Model([noise, label], [valid, target_label]) self.combined.compile(loss=losses, optimizer=optimizer) def build_generator(self): model = Sequential() model.add(Dense(128 * 7 * 7, activation="relu", input_dim=self.latent_dim)) model.add(Reshape((7, 7, 128))) model.add(BatchNormalization(momentum=0.8)) model.add(UpSampling2D()) model.add(Conv2D(128, kernel_size=3, padding="same")) model.add(Activation("relu")) model.add(BatchNormalization(momentum=0.8)) model.add(UpSampling2D()) model.add(Conv2D(64, kernel_size=3, padding="same")) model.add(Activation("relu")) model.add(BatchNormalization(momentum=0.8)) model.add(Conv2D(self.channels, kernel_size=3, padding='same')) model.add(Activation("tanh")) model.summary() noise = Input(shape=(self.latent_dim,)) label = Input(shape=(1,), dtype='int32') label_embedding = Flatten()(Embedding(self.num_classes, self.latent_dim)(label)) model_input = multiply([noise, label_embedding]) img = model(model_input) return Model([noise, label], img) def build_discriminator(self): model = Sequential() model.add(Conv2D(16, kernel_size=3, strides=2, input_shape=self.img_shape, padding="same")) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Conv2D(32, kernel_size=3, strides=2, padding="same")) model.add(ZeroPadding2D(padding=((0,1),(0,1)))) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(BatchNormalization(momentum=0.8)) model.add(Conv2D(64, kernel_size=3, strides=2, padding="same")) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(BatchNormalization(momentum=0.8)) model.add(Conv2D(128, kernel_size=3, strides=1, padding="same")) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Flatten()) model.summary() img = Input(shape=self.img_shape) # Extract feature representation features = model(img) # Determine validity and label of the image validity = Dense(1, activation="sigmoid")(features) label = Dense(self.num_classes, activation="softmax")(features) return Model(img, [validity, label]) def train(self, epochs, batch_size=128, sample_interval=50): # Load the dataset (X_train, y_train), (_, _) = mnist.load_data() # Configure inputs X_train = (X_train.astype(np.float32) - 127.5) / 127.5 X_train = np.expand_dims(X_train, axis=3) y_train = y_train.reshape(-1, 1) # Adversarial ground truths valid = np.ones((batch_size, 1)) fake = np.zeros((batch_size, 1)) for epoch in range(epochs): # --------------------- # Train Discriminator # --------------------- # Select a random batch of images idx = np.random.randint(0, X_train.shape[0], batch_size) imgs = X_train[idx] # Sample noise as generator input noise = np.random.normal(0, 1, (batch_size, self.latent_dim)) # The labels of the digits that the generator tries to create an # image representation of sampled_labels = np.random.randint(0, 10, (batch_size, 1)) # Generate a half batch of new images gen_imgs = self.generator.predict([noise, sampled_labels]) # Image labels. 0-9 img_labels = y_train[idx] # Train the discriminator d_loss_real = self.discriminator.train_on_batch(imgs, [valid, img_labels]) d_loss_fake = self.discriminator.train_on_batch(gen_imgs, [fake, sampled_labels]) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # --------------------- # Train Generator # --------------------- # Train the generator g_loss = self.combined.train_on_batch([noise, sampled_labels], [valid, sampled_labels]) # Plot the progress print ("%d [D loss: %f, acc.: %.2f%%, op_acc: %.2f%%] [G loss: %f]" % (epoch, d_loss[0], 100*d_loss[3], 100*d_loss[4], g_loss[0])) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.save_model() self.sample_images(epoch) def sample_images(self, epoch): r, c = 10, 10 noise = np.random.normal(0, 1, (r * c, self.latent_dim)) sampled_labels = np.array([num for _ in range(r) for num in range(c)]) gen_imgs = self.generator.predict([noise, sampled_labels]) # Rescale images 0 - 1 gen_imgs = 0.5 * gen_imgs + 0.5 fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): axs[i,j].imshow(gen_imgs[cnt,:,:,0], cmap='gray') axs[i,j].axis('off') cnt += 1 fig.savefig("images/%d.png" % epoch) plt.close() def save_model(self): def save(model, model_name): model_path = "saved_model/%s.json" % model_name weights_path = "saved_model/%s_weights.hdf5" % model_name options = {"file_arch": model_path, "file_weight": weights_path} json_string = model.to_json() open(options['file_arch'], 'w').write(json_string) model.save_weights(options['file_weight']) save(self.generator, "generator") save(self.discriminator, "discriminator") if __name__ == '__main__': acgan = ACGAN() acgan.train(epochs=14000, batch_size=32, sample_interval=200) ================================================ FILE: acgan/images/.gitignore ================================================ * !.gitignore ================================================ FILE: acgan/saved_model/.gitignore ================================================ * !.gitignore ================================================ FILE: bgan/bgan.py ================================================ from __future__ import print_function, division from keras.datasets import mnist from keras.layers import Input, Dense, Reshape, Flatten, Dropout from keras.layers import BatchNormalization, Activation, ZeroPadding2D from keras.layers.advanced_activations import LeakyReLU from keras.layers.convolutional import UpSampling2D, Conv2D from keras.models import Sequential, Model from keras.optimizers import Adam import keras.backend as K import matplotlib.pyplot as plt import sys import numpy as np class BGAN(): """Reference: https://wiseodd.github.io/techblog/2017/03/07/boundary-seeking-gan/""" def __init__(self): self.img_rows = 28 self.img_cols = 28 self.channels = 1 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.latent_dim = 100 optimizer = Adam(0.0002, 0.5) # Build and compile the discriminator self.discriminator = self.build_discriminator() self.discriminator.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) # Build the generator self.generator = self.build_generator() # The generator takes noise as input and generated imgs z = Input(shape=(self.latent_dim,)) img = self.generator(z) # For the combined model we will only train the generator self.discriminator.trainable = False # The valid takes generated images as input and determines validity valid = self.discriminator(img) # The combined model (stacked generator and discriminator) # Trains the generator to fool the discriminator self.combined = Model(z, valid) self.combined.compile(loss=self.boundary_loss, optimizer=optimizer) def build_generator(self): model = Sequential() model.add(Dense(256, input_dim=self.latent_dim)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(1024)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(np.prod(self.img_shape), activation='tanh')) model.add(Reshape(self.img_shape)) model.summary() noise = Input(shape=(self.latent_dim,)) img = model(noise) return Model(noise, img) def build_discriminator(self): model = Sequential() model.add(Flatten(input_shape=self.img_shape)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(256)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(1, activation='sigmoid')) model.summary() img = Input(shape=self.img_shape) validity = model(img) return Model(img, validity) def boundary_loss(self, y_true, y_pred): """ Boundary seeking loss. Reference: https://wiseodd.github.io/techblog/2017/03/07/boundary-seeking-gan/ """ return 0.5 * K.mean((K.log(y_pred) - K.log(1 - y_pred))**2) def train(self, epochs, batch_size=128, sample_interval=50): # Load the dataset (X_train, _), (_, _) = mnist.load_data() # Rescale -1 to 1 X_train = X_train / 127.5 - 1. X_train = np.expand_dims(X_train, axis=3) # Adversarial ground truths valid = np.ones((batch_size, 1)) fake = np.zeros((batch_size, 1)) for epoch in range(epochs): # --------------------- # Train Discriminator # --------------------- # Select a random batch of images idx = np.random.randint(0, X_train.shape[0], batch_size) imgs = X_train[idx] noise = np.random.normal(0, 1, (batch_size, self.latent_dim)) # Generate a batch of new images gen_imgs = self.generator.predict(noise) # Train the discriminator d_loss_real = self.discriminator.train_on_batch(imgs, valid) d_loss_fake = self.discriminator.train_on_batch(gen_imgs, fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # --------------------- # Train Generator # --------------------- g_loss = self.combined.train_on_batch(noise, valid) # Plot the progress print ("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]" % (epoch, d_loss[0], 100*d_loss[1], g_loss)) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.sample_images(epoch) def sample_images(self, epoch): r, c = 5, 5 noise = np.random.normal(0, 1, (r * c, self.latent_dim)) gen_imgs = self.generator.predict(noise) # Rescale images 0 - 1 gen_imgs = 0.5 * gen_imgs + 0.5 fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): axs[i,j].imshow(gen_imgs[cnt, :,:,0], cmap='gray') axs[i,j].axis('off') cnt += 1 fig.savefig("images/mnist_%d.png" % epoch) plt.close() if __name__ == '__main__': bgan = BGAN() bgan.train(epochs=30000, batch_size=32, sample_interval=200) ================================================ FILE: bgan/images/.gitignore ================================================ * !.gitignore ================================================ FILE: bgan/saved_model/.gitignore ================================================ * !.gitignore ================================================ FILE: bigan/bigan.py ================================================ from __future__ import print_function, division from keras.datasets import mnist from keras.layers import Input, Dense, Reshape, Flatten, Dropout, multiply, GaussianNoise from keras.layers import BatchNormalization, Activation, Embedding, ZeroPadding2D from keras.layers import MaxPooling2D, concatenate from keras.layers.advanced_activations import LeakyReLU from keras.layers.convolutional import UpSampling2D, Conv2D from keras.models import Sequential, Model from keras.optimizers import Adam from keras import losses from keras.utils import to_categorical import keras.backend as K import matplotlib.pyplot as plt import numpy as np class BIGAN(): def __init__(self): self.img_rows = 28 self.img_cols = 28 self.channels = 1 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.latent_dim = 100 optimizer = Adam(0.0002, 0.5) # Build and compile the discriminator self.discriminator = self.build_discriminator() self.discriminator.compile(loss=['binary_crossentropy'], optimizer=optimizer, metrics=['accuracy']) # Build the generator self.generator = self.build_generator() # Build the encoder self.encoder = self.build_encoder() # The part of the bigan that trains the discriminator and encoder self.discriminator.trainable = False # Generate image from sampled noise z = Input(shape=(self.latent_dim, )) img_ = self.generator(z) # Encode image img = Input(shape=self.img_shape) z_ = self.encoder(img) # Latent -> img is fake, and img -> latent is valid fake = self.discriminator([z, img_]) valid = self.discriminator([z_, img]) # Set up and compile the combined model # Trains generator to fool the discriminator self.bigan_generator = Model([z, img], [fake, valid]) self.bigan_generator.compile(loss=['binary_crossentropy', 'binary_crossentropy'], optimizer=optimizer) def build_encoder(self): model = Sequential() model.add(Flatten(input_shape=self.img_shape)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(self.latent_dim)) model.summary() img = Input(shape=self.img_shape) z = model(img) return Model(img, z) def build_generator(self): model = Sequential() model.add(Dense(512, input_dim=self.latent_dim)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(np.prod(self.img_shape), activation='tanh')) model.add(Reshape(self.img_shape)) model.summary() z = Input(shape=(self.latent_dim,)) gen_img = model(z) return Model(z, gen_img) def build_discriminator(self): z = Input(shape=(self.latent_dim, )) img = Input(shape=self.img_shape) d_in = concatenate([z, Flatten()(img)]) model = Dense(1024)(d_in) model = LeakyReLU(alpha=0.2)(model) model = Dropout(0.5)(model) model = Dense(1024)(model) model = LeakyReLU(alpha=0.2)(model) model = Dropout(0.5)(model) model = Dense(1024)(model) model = LeakyReLU(alpha=0.2)(model) model = Dropout(0.5)(model) validity = Dense(1, activation="sigmoid")(model) return Model([z, img], validity) def train(self, epochs, batch_size=128, sample_interval=50): # Load the dataset (X_train, _), (_, _) = mnist.load_data() # Rescale -1 to 1 X_train = (X_train.astype(np.float32) - 127.5) / 127.5 X_train = np.expand_dims(X_train, axis=3) # Adversarial ground truths valid = np.ones((batch_size, 1)) fake = np.zeros((batch_size, 1)) for epoch in range(epochs): # --------------------- # Train Discriminator # --------------------- # Sample noise and generate img z = np.random.normal(size=(batch_size, self.latent_dim)) imgs_ = self.generator.predict(z) # Select a random batch of images and encode idx = np.random.randint(0, X_train.shape[0], batch_size) imgs = X_train[idx] z_ = self.encoder.predict(imgs) # Train the discriminator (img -> z is valid, z -> img is fake) d_loss_real = self.discriminator.train_on_batch([z_, imgs], valid) d_loss_fake = self.discriminator.train_on_batch([z, imgs_], fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # --------------------- # Train Generator # --------------------- # Train the generator (z -> img is valid and img -> z is is invalid) g_loss = self.bigan_generator.train_on_batch([z, imgs], [valid, fake]) # Plot the progress print ("%d [D loss: %f, acc: %.2f%%] [G loss: %f]" % (epoch, d_loss[0], 100*d_loss[1], g_loss[0])) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.sample_interval(epoch) def sample_interval(self, epoch): r, c = 5, 5 z = np.random.normal(size=(25, self.latent_dim)) gen_imgs = self.generator.predict(z) gen_imgs = 0.5 * gen_imgs + 0.5 fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): axs[i,j].imshow(gen_imgs[cnt, :,:,0], cmap='gray') axs[i,j].axis('off') cnt += 1 fig.savefig("images/mnist_%d.png" % epoch) plt.close() if __name__ == '__main__': bigan = BIGAN() bigan.train(epochs=40000, batch_size=32, sample_interval=400) ================================================ FILE: bigan/images/.gitignore ================================================ * !.gitignore ================================================ FILE: bigan/saved_model/.gitignore ================================================ * !.gitignore ================================================ FILE: ccgan/ccgan.py ================================================ from __future__ import print_function, division from keras.datasets import mnist from keras_contrib.layers.normalization.instancenormalization import InstanceNormalization from keras.layers import Input, Dense, Reshape, Flatten, Dropout, multiply, GaussianNoise from keras.layers import BatchNormalization, Activation, Embedding, ZeroPadding2D from keras.layers import Concatenate from keras.layers.advanced_activations import LeakyReLU from keras.layers.convolutional import UpSampling2D, Conv2D from keras.models import Sequential, Model from keras.optimizers import Adam from keras import losses from keras.utils import to_categorical import keras.backend as K import scipy import matplotlib.pyplot as plt import numpy as np class CCGAN(): def __init__(self): self.img_rows = 32 self.img_cols = 32 self.channels = 1 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.mask_height = 10 self.mask_width = 10 self.num_classes = 10 # Number of filters in first layer of generator and discriminator self.gf = 32 self.df = 32 optimizer = Adam(0.0002, 0.5) # Build and compile the discriminator self.discriminator = self.build_discriminator() self.discriminator.compile(loss=['mse', 'categorical_crossentropy'], loss_weights=[0.5, 0.5], optimizer=optimizer, metrics=['accuracy']) # Build the generator self.generator = self.build_generator() # The generator takes noise as input and generates imgs masked_img = Input(shape=self.img_shape) gen_img = self.generator(masked_img) # For the combined model we will only train the generator self.discriminator.trainable = False # The valid takes generated images as input and determines validity valid, _ = self.discriminator(gen_img) # The combined model (stacked generator and discriminator) # Trains the generator to fool the discriminator self.combined = Model(masked_img , valid) self.combined.compile(loss=['mse'], optimizer=optimizer) def build_generator(self): """U-Net Generator""" def conv2d(layer_input, filters, f_size=4, bn=True): """Layers used during downsampling""" d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input) d = LeakyReLU(alpha=0.2)(d) if bn: d = BatchNormalization(momentum=0.8)(d) return d def deconv2d(layer_input, skip_input, filters, f_size=4, dropout_rate=0): """Layers used during upsampling""" u = UpSampling2D(size=2)(layer_input) u = Conv2D(filters, kernel_size=f_size, strides=1, padding='same', activation='relu')(u) if dropout_rate: u = Dropout(dropout_rate)(u) u = BatchNormalization(momentum=0.8)(u) u = Concatenate()([u, skip_input]) return u img = Input(shape=self.img_shape) # Downsampling d1 = conv2d(img, self.gf, bn=False) d2 = conv2d(d1, self.gf*2) d3 = conv2d(d2, self.gf*4) d4 = conv2d(d3, self.gf*8) # Upsampling u1 = deconv2d(d4, d3, self.gf*4) u2 = deconv2d(u1, d2, self.gf*2) u3 = deconv2d(u2, d1, self.gf) u4 = UpSampling2D(size=2)(u3) output_img = Conv2D(self.channels, kernel_size=4, strides=1, padding='same', activation='tanh')(u4) return Model(img, output_img) def build_discriminator(self): img = Input(shape=self.img_shape) model = Sequential() model.add(Conv2D(64, kernel_size=4, strides=2, padding='same', input_shape=self.img_shape)) model.add(LeakyReLU(alpha=0.8)) model.add(Conv2D(128, kernel_size=4, strides=2, padding='same')) model.add(LeakyReLU(alpha=0.2)) model.add(InstanceNormalization()) model.add(Conv2D(256, kernel_size=4, strides=2, padding='same')) model.add(LeakyReLU(alpha=0.2)) model.add(InstanceNormalization()) model.summary() img = Input(shape=self.img_shape) features = model(img) validity = Conv2D(1, kernel_size=4, strides=1, padding='same')(features) label = Flatten()(features) label = Dense(self.num_classes+1, activation="softmax")(label) return Model(img, [validity, label]) def mask_randomly(self, imgs): y1 = np.random.randint(0, self.img_rows - self.mask_height, imgs.shape[0]) y2 = y1 + self.mask_height x1 = np.random.randint(0, self.img_rows - self.mask_width, imgs.shape[0]) x2 = x1 + self.mask_width masked_imgs = np.empty_like(imgs) for i, img in enumerate(imgs): masked_img = img.copy() _y1, _y2, _x1, _x2 = y1[i], y2[i], x1[i], x2[i], masked_img[_y1:_y2, _x1:_x2, :] = 0 masked_imgs[i] = masked_img return masked_imgs def train(self, epochs, batch_size=128, sample_interval=50): # Load the dataset (X_train, y_train), (_, _) = mnist.load_data() # Rescale MNIST to 32x32 X_train = np.array([scipy.misc.imresize(x, [self.img_rows, self.img_cols]) for x in X_train]) # Rescale -1 to 1 X_train = (X_train.astype(np.float32) - 127.5) / 127.5 X_train = np.expand_dims(X_train, axis=3) y_train = y_train.reshape(-1, 1) # Adversarial ground truths valid = np.ones((batch_size, 4, 4, 1)) fake = np.zeros((batch_size, 4, 4, 1)) for epoch in range(epochs): # --------------------- # Train Discriminator # --------------------- # Sample half batch of images idx = np.random.randint(0, X_train.shape[0], batch_size) imgs = X_train[idx] labels = y_train[idx] masked_imgs = self.mask_randomly(imgs) # Generate a half batch of new images gen_imgs = self.generator.predict(masked_imgs) # One-hot encoding of labels labels = to_categorical(labels, num_classes=self.num_classes+1) fake_labels = to_categorical(np.full((batch_size, 1), self.num_classes), num_classes=self.num_classes+1) # Train the discriminator d_loss_real = self.discriminator.train_on_batch(imgs, [valid, labels]) d_loss_fake = self.discriminator.train_on_batch(gen_imgs, [fake, fake_labels]) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # --------------------- # Train Generator # --------------------- # Train the generator g_loss = self.combined.train_on_batch(masked_imgs, valid) # Plot the progress print ("%d [D loss: %f, op_acc: %.2f%%] [G loss: %f]" % (epoch, d_loss[0], 100*d_loss[4], g_loss)) # If at save interval => save generated image samples if epoch % sample_interval == 0: # Select a random half batch of images idx = np.random.randint(0, X_train.shape[0], 6) imgs = X_train[idx] self.sample_images(epoch, imgs) self.save_model() def sample_images(self, epoch, imgs): r, c = 3, 6 masked_imgs = self.mask_randomly(imgs) gen_imgs = self.generator.predict(masked_imgs) imgs = (imgs + 1.0) * 0.5 masked_imgs = (masked_imgs + 1.0) * 0.5 gen_imgs = (gen_imgs + 1.0) * 0.5 gen_imgs = np.where(gen_imgs < 0, 0, gen_imgs) fig, axs = plt.subplots(r, c) for i in range(c): axs[0,i].imshow(imgs[i, :, :, 0], cmap='gray') axs[0,i].axis('off') axs[1,i].imshow(masked_imgs[i, :, :, 0], cmap='gray') axs[1,i].axis('off') axs[2,i].imshow(gen_imgs[i, :, :, 0], cmap='gray') axs[2,i].axis('off') fig.savefig("images/%d.png" % epoch) plt.close() def save_model(self): def save(model, model_name): model_path = "saved_model/%s.json" % model_name weights_path = "saved_model/%s_weights.hdf5" % model_name options = {"file_arch": model_path, "file_weight": weights_path} json_string = model.to_json() open(options['file_arch'], 'w').write(json_string) model.save_weights(options['file_weight']) save(self.generator, "ccgan_generator") save(self.discriminator, "ccgan_discriminator") if __name__ == '__main__': ccgan = CCGAN() ccgan.train(epochs=20000, batch_size=32, sample_interval=200) ================================================ FILE: ccgan/images/.gitignore ================================================ * !.gitignore ================================================ FILE: ccgan/saved_model/.gitignore ================================================ * !.gitignore ================================================ FILE: cgan/cgan.py ================================================ from __future__ import print_function, division from keras.datasets import mnist from keras.layers import Input, Dense, Reshape, Flatten, Dropout, multiply from keras.layers import BatchNormalization, Activation, Embedding, ZeroPadding2D from keras.layers.advanced_activations import LeakyReLU from keras.layers.convolutional import UpSampling2D, Conv2D from keras.models import Sequential, Model from keras.optimizers import Adam import matplotlib.pyplot as plt import numpy as np class CGAN(): def __init__(self): # Input shape self.img_rows = 28 self.img_cols = 28 self.channels = 1 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.num_classes = 10 self.latent_dim = 100 optimizer = Adam(0.0002, 0.5) # Build and compile the discriminator self.discriminator = self.build_discriminator() self.discriminator.compile(loss=['binary_crossentropy'], optimizer=optimizer, metrics=['accuracy']) # Build the generator self.generator = self.build_generator() # The generator takes noise and the target label as input # and generates the corresponding digit of that label noise = Input(shape=(self.latent_dim,)) label = Input(shape=(1,)) img = self.generator([noise, label]) # For the combined model we will only train the generator self.discriminator.trainable = False # The discriminator takes generated image as input and determines validity # and the label of that image valid = self.discriminator([img, label]) # The combined model (stacked generator and discriminator) # Trains generator to fool discriminator self.combined = Model([noise, label], valid) self.combined.compile(loss=['binary_crossentropy'], optimizer=optimizer) def build_generator(self): model = Sequential() model.add(Dense(256, input_dim=self.latent_dim)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(1024)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(np.prod(self.img_shape), activation='tanh')) model.add(Reshape(self.img_shape)) model.summary() noise = Input(shape=(self.latent_dim,)) label = Input(shape=(1,), dtype='int32') label_embedding = Flatten()(Embedding(self.num_classes, self.latent_dim)(label)) model_input = multiply([noise, label_embedding]) img = model(model_input) return Model([noise, label], img) def build_discriminator(self): model = Sequential() model.add(Dense(512, input_dim=np.prod(self.img_shape))) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.4)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.4)) model.add(Dense(1, activation='sigmoid')) model.summary() img = Input(shape=self.img_shape) label = Input(shape=(1,), dtype='int32') label_embedding = Flatten()(Embedding(self.num_classes, np.prod(self.img_shape))(label)) flat_img = Flatten()(img) model_input = multiply([flat_img, label_embedding]) validity = model(model_input) return Model([img, label], validity) def train(self, epochs, batch_size=128, sample_interval=50): # Load the dataset (X_train, y_train), (_, _) = mnist.load_data() # Configure input X_train = (X_train.astype(np.float32) - 127.5) / 127.5 X_train = np.expand_dims(X_train, axis=3) y_train = y_train.reshape(-1, 1) # Adversarial ground truths valid = np.ones((batch_size, 1)) fake = np.zeros((batch_size, 1)) for epoch in range(epochs): # --------------------- # Train Discriminator # --------------------- # Select a random half batch of images idx = np.random.randint(0, X_train.shape[0], batch_size) imgs, labels = X_train[idx], y_train[idx] # Sample noise as generator input noise = np.random.normal(0, 1, (batch_size, 100)) # Generate a half batch of new images gen_imgs = self.generator.predict([noise, labels]) # Train the discriminator d_loss_real = self.discriminator.train_on_batch([imgs, labels], valid) d_loss_fake = self.discriminator.train_on_batch([gen_imgs, labels], fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # --------------------- # Train Generator # --------------------- # Condition on labels sampled_labels = np.random.randint(0, 10, batch_size).reshape(-1, 1) # Train the generator g_loss = self.combined.train_on_batch([noise, sampled_labels], valid) # Plot the progress print ("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]" % (epoch, d_loss[0], 100*d_loss[1], g_loss)) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.sample_images(epoch) def sample_images(self, epoch): r, c = 2, 5 noise = np.random.normal(0, 1, (r * c, 100)) sampled_labels = np.arange(0, 10).reshape(-1, 1) gen_imgs = self.generator.predict([noise, sampled_labels]) # Rescale images 0 - 1 gen_imgs = 0.5 * gen_imgs + 0.5 fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): axs[i,j].imshow(gen_imgs[cnt,:,:,0], cmap='gray') axs[i,j].set_title("Digit: %d" % sampled_labels[cnt]) axs[i,j].axis('off') cnt += 1 fig.savefig("images/%d.png" % epoch) plt.close() if __name__ == '__main__': cgan = CGAN() cgan.train(epochs=20000, batch_size=32, sample_interval=200) ================================================ FILE: cgan/images/.gitignore ================================================ * !.gitignore ================================================ FILE: cgan/saved_model/.gitignore ================================================ * !.gitignore ================================================ FILE: cogan/cogan.py ================================================ from __future__ import print_function, division import scipy from keras.datasets import mnist from keras.layers import Input, Dense, Reshape, Flatten, Dropout from keras.layers import BatchNormalization, Activation, ZeroPadding2D from keras.layers.advanced_activations import LeakyReLU from keras.layers.convolutional import UpSampling2D, Conv2D from keras.models import Sequential, Model from keras.optimizers import Adam import matplotlib.pyplot as plt import sys import numpy as np class COGAN(): """Reference: https://wiseodd.github.io/techblog/2017/02/18/coupled_gan/""" def __init__(self): self.img_rows = 28 self.img_cols = 28 self.channels = 1 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.latent_dim = 100 optimizer = Adam(0.0002, 0.5) # Build and compile the discriminator self.d1, self.d2 = self.build_discriminators() self.d1.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) self.d2.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) # Build the generator self.g1, self.g2 = self.build_generators() # The generator takes noise as input and generated imgs z = Input(shape=(self.latent_dim,)) img1 = self.g1(z) img2 = self.g2(z) # For the combined model we will only train the generators self.d1.trainable = False self.d2.trainable = False # The valid takes generated images as input and determines validity valid1 = self.d1(img1) valid2 = self.d2(img2) # The combined model (stacked generators and discriminators) # Trains generators to fool discriminators self.combined = Model(z, [valid1, valid2]) self.combined.compile(loss=['binary_crossentropy', 'binary_crossentropy'], optimizer=optimizer) def build_generators(self): # Shared weights between generators model = Sequential() model.add(Dense(256, input_dim=self.latent_dim)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) noise = Input(shape=(self.latent_dim,)) feature_repr = model(noise) # Generator 1 g1 = Dense(1024)(feature_repr) g1 = LeakyReLU(alpha=0.2)(g1) g1 = BatchNormalization(momentum=0.8)(g1) g1 = Dense(np.prod(self.img_shape), activation='tanh')(g1) img1 = Reshape(self.img_shape)(g1) # Generator 2 g2 = Dense(1024)(feature_repr) g2 = LeakyReLU(alpha=0.2)(g2) g2 = BatchNormalization(momentum=0.8)(g2) g2 = Dense(np.prod(self.img_shape), activation='tanh')(g2) img2 = Reshape(self.img_shape)(g2) model.summary() return Model(noise, img1), Model(noise, img2) def build_discriminators(self): img1 = Input(shape=self.img_shape) img2 = Input(shape=self.img_shape) # Shared discriminator layers model = Sequential() model.add(Flatten(input_shape=self.img_shape)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(256)) model.add(LeakyReLU(alpha=0.2)) img1_embedding = model(img1) img2_embedding = model(img2) # Discriminator 1 validity1 = Dense(1, activation='sigmoid')(img1_embedding) # Discriminator 2 validity2 = Dense(1, activation='sigmoid')(img2_embedding) return Model(img1, validity1), Model(img2, validity2) def train(self, epochs, batch_size=128, sample_interval=50): # Load the dataset (X_train, _), (_, _) = mnist.load_data() # Rescale -1 to 1 X_train = (X_train.astype(np.float32) - 127.5) / 127.5 X_train = np.expand_dims(X_train, axis=3) # Images in domain A and B (rotated) X1 = X_train[:int(X_train.shape[0]/2)] X2 = X_train[int(X_train.shape[0]/2):] X2 = scipy.ndimage.interpolation.rotate(X2, 90, axes=(1, 2)) # Adversarial ground truths valid = np.ones((batch_size, 1)) fake = np.zeros((batch_size, 1)) for epoch in range(epochs): # ---------------------- # Train Discriminators # ---------------------- # Select a random batch of images idx = np.random.randint(0, X1.shape[0], batch_size) imgs1 = X1[idx] imgs2 = X2[idx] # Sample noise as generator input noise = np.random.normal(0, 1, (batch_size, 100)) # Generate a batch of new images gen_imgs1 = self.g1.predict(noise) gen_imgs2 = self.g2.predict(noise) # Train the discriminators d1_loss_real = self.d1.train_on_batch(imgs1, valid) d2_loss_real = self.d2.train_on_batch(imgs2, valid) d1_loss_fake = self.d1.train_on_batch(gen_imgs1, fake) d2_loss_fake = self.d2.train_on_batch(gen_imgs2, fake) d1_loss = 0.5 * np.add(d1_loss_real, d1_loss_fake) d2_loss = 0.5 * np.add(d2_loss_real, d2_loss_fake) # ------------------ # Train Generators # ------------------ g_loss = self.combined.train_on_batch(noise, [valid, valid]) # Plot the progress print ("%d [D1 loss: %f, acc.: %.2f%%] [D2 loss: %f, acc.: %.2f%%] [G loss: %f]" \ % (epoch, d1_loss[0], 100*d1_loss[1], d2_loss[0], 100*d2_loss[1], g_loss[0])) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.sample_images(epoch) def sample_images(self, epoch): r, c = 4, 4 noise = np.random.normal(0, 1, (r * int(c/2), 100)) gen_imgs1 = self.g1.predict(noise) gen_imgs2 = self.g2.predict(noise) gen_imgs = np.concatenate([gen_imgs1, gen_imgs2]) # Rescale images 0 - 1 gen_imgs = 0.5 * gen_imgs + 0.5 fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): axs[i,j].imshow(gen_imgs[cnt, :,:,0], cmap='gray') axs[i,j].axis('off') cnt += 1 fig.savefig("images/mnist_%d.png" % epoch) plt.close() if __name__ == '__main__': gan = COGAN() gan.train(epochs=30000, batch_size=32, sample_interval=200) ================================================ FILE: cogan/images/.gitignore ================================================ * !.gitignore ================================================ FILE: cogan/saved_model/.gitignore ================================================ * !.gitignore ================================================ FILE: context_encoder/context_encoder.py ================================================ from __future__ import print_function, division from keras.datasets import cifar10 from keras.layers import Input, Dense, Reshape, Flatten, Dropout, multiply, GaussianNoise from keras.layers import BatchNormalization, Activation, Embedding, ZeroPadding2D from keras.layers import MaxPooling2D from keras.layers.advanced_activations import LeakyReLU from keras.layers.convolutional import UpSampling2D, Conv2D from keras.models import Sequential, Model from keras.optimizers import Adam from keras import losses from keras.utils import to_categorical import keras.backend as K import matplotlib.pyplot as plt import numpy as np class ContextEncoder(): def __init__(self): self.img_rows = 32 self.img_cols = 32 self.mask_height = 8 self.mask_width = 8 self.channels = 3 self.num_classes = 2 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.missing_shape = (self.mask_height, self.mask_width, self.channels) optimizer = Adam(0.0002, 0.5) # Build and compile the discriminator self.discriminator = self.build_discriminator() self.discriminator.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) # Build the generator self.generator = self.build_generator() # The generator takes noise as input and generates the missing # part of the image masked_img = Input(shape=self.img_shape) gen_missing = self.generator(masked_img) # For the combined model we will only train the generator self.discriminator.trainable = False # The discriminator takes generated images as input and determines # if it is generated or if it is a real image valid = self.discriminator(gen_missing) # The combined model (stacked generator and discriminator) # Trains generator to fool discriminator self.combined = Model(masked_img , [gen_missing, valid]) self.combined.compile(loss=['mse', 'binary_crossentropy'], loss_weights=[0.999, 0.001], optimizer=optimizer) def build_generator(self): model = Sequential() # Encoder model.add(Conv2D(32, kernel_size=3, strides=2, input_shape=self.img_shape, padding="same")) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Conv2D(64, kernel_size=3, strides=2, padding="same")) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Conv2D(128, kernel_size=3, strides=2, padding="same")) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Conv2D(512, kernel_size=1, strides=2, padding="same")) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.5)) # Decoder model.add(UpSampling2D()) model.add(Conv2D(128, kernel_size=3, padding="same")) model.add(Activation('relu')) model.add(BatchNormalization(momentum=0.8)) model.add(UpSampling2D()) model.add(Conv2D(64, kernel_size=3, padding="same")) model.add(Activation('relu')) model.add(BatchNormalization(momentum=0.8)) model.add(Conv2D(self.channels, kernel_size=3, padding="same")) model.add(Activation('tanh')) model.summary() masked_img = Input(shape=self.img_shape) gen_missing = model(masked_img) return Model(masked_img, gen_missing) def build_discriminator(self): model = Sequential() model.add(Conv2D(64, kernel_size=3, strides=2, input_shape=self.missing_shape, padding="same")) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Conv2D(128, kernel_size=3, strides=2, padding="same")) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Conv2D(256, kernel_size=3, padding="same")) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Flatten()) model.add(Dense(1, activation='sigmoid')) model.summary() img = Input(shape=self.missing_shape) validity = model(img) return Model(img, validity) def mask_randomly(self, imgs): y1 = np.random.randint(0, self.img_rows - self.mask_height, imgs.shape[0]) y2 = y1 + self.mask_height x1 = np.random.randint(0, self.img_rows - self.mask_width, imgs.shape[0]) x2 = x1 + self.mask_width masked_imgs = np.empty_like(imgs) missing_parts = np.empty((imgs.shape[0], self.mask_height, self.mask_width, self.channels)) for i, img in enumerate(imgs): masked_img = img.copy() _y1, _y2, _x1, _x2 = y1[i], y2[i], x1[i], x2[i] missing_parts[i] = masked_img[_y1:_y2, _x1:_x2, :].copy() masked_img[_y1:_y2, _x1:_x2, :] = 0 masked_imgs[i] = masked_img return masked_imgs, missing_parts, (y1, y2, x1, x2) def train(self, epochs, batch_size=128, sample_interval=50): # Load the dataset (X_train, y_train), (_, _) = cifar10.load_data() # Extract dogs and cats X_cats = X_train[(y_train == 3).flatten()] X_dogs = X_train[(y_train == 5).flatten()] X_train = np.vstack((X_cats, X_dogs)) # Rescale -1 to 1 X_train = X_train / 127.5 - 1. y_train = y_train.reshape(-1, 1) # Adversarial ground truths valid = np.ones((batch_size, 1)) fake = np.zeros((batch_size, 1)) for epoch in range(epochs): # --------------------- # Train Discriminator # --------------------- # Select a random batch of images idx = np.random.randint(0, X_train.shape[0], batch_size) imgs = X_train[idx] masked_imgs, missing_parts, _ = self.mask_randomly(imgs) # Generate a batch of new images gen_missing = self.generator.predict(masked_imgs) # Train the discriminator d_loss_real = self.discriminator.train_on_batch(missing_parts, valid) d_loss_fake = self.discriminator.train_on_batch(gen_missing, fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # --------------------- # Train Generator # --------------------- g_loss = self.combined.train_on_batch(masked_imgs, [missing_parts, valid]) # Plot the progress print ("%d [D loss: %f, acc: %.2f%%] [G loss: %f, mse: %f]" % (epoch, d_loss[0], 100*d_loss[1], g_loss[0], g_loss[1])) # If at save interval => save generated image samples if epoch % sample_interval == 0: idx = np.random.randint(0, X_train.shape[0], 6) imgs = X_train[idx] self.sample_images(epoch, imgs) def sample_images(self, epoch, imgs): r, c = 3, 6 masked_imgs, missing_parts, (y1, y2, x1, x2) = self.mask_randomly(imgs) gen_missing = self.generator.predict(masked_imgs) imgs = 0.5 * imgs + 0.5 masked_imgs = 0.5 * masked_imgs + 0.5 gen_missing = 0.5 * gen_missing + 0.5 fig, axs = plt.subplots(r, c) for i in range(c): axs[0,i].imshow(imgs[i, :,:]) axs[0,i].axis('off') axs[1,i].imshow(masked_imgs[i, :,:]) axs[1,i].axis('off') filled_in = imgs[i].copy() filled_in[y1[i]:y2[i], x1[i]:x2[i], :] = gen_missing[i] axs[2,i].imshow(filled_in) axs[2,i].axis('off') fig.savefig("images/%d.png" % epoch) plt.close() def save_model(self): def save(model, model_name): model_path = "saved_model/%s.json" % model_name weights_path = "saved_model/%s_weights.hdf5" % model_name options = {"file_arch": model_path, "file_weight": weights_path} json_string = model.to_json() open(options['file_arch'], 'w').write(json_string) model.save_weights(options['file_weight']) save(self.generator, "generator") save(self.discriminator, "discriminator") if __name__ == '__main__': context_encoder = ContextEncoder() context_encoder.train(epochs=30000, batch_size=64, sample_interval=50) ================================================ FILE: context_encoder/images/.gitignore ================================================ * !.gitignore ================================================ FILE: context_encoder/saved_model/.gitignore ================================================ * !.gitignore ================================================ FILE: cyclegan/cyclegan.py ================================================ from __future__ import print_function, division import scipy from keras.datasets import mnist from keras_contrib.layers.normalization.instancenormalization import InstanceNormalization from keras.layers import Input, Dense, Reshape, Flatten, Dropout, Concatenate from keras.layers import BatchNormalization, Activation, ZeroPadding2D from keras.layers.advanced_activations import LeakyReLU from keras.layers.convolutional import UpSampling2D, Conv2D from keras.models import Sequential, Model from keras.optimizers import Adam import datetime import matplotlib.pyplot as plt import sys from data_loader import DataLoader import numpy as np import os class CycleGAN(): def __init__(self): # Input shape self.img_rows = 128 self.img_cols = 128 self.channels = 3 self.img_shape = (self.img_rows, self.img_cols, self.channels) # Configure data loader self.dataset_name = 'apple2orange' self.data_loader = DataLoader(dataset_name=self.dataset_name, img_res=(self.img_rows, self.img_cols)) # Calculate output shape of D (PatchGAN) patch = int(self.img_rows / 2**4) self.disc_patch = (patch, patch, 1) # Number of filters in the first layer of G and D self.gf = 32 self.df = 64 # Loss weights self.lambda_cycle = 10.0 # Cycle-consistency loss self.lambda_id = 0.1 * self.lambda_cycle # Identity loss optimizer = Adam(0.0002, 0.5) # Build and compile the discriminators self.d_A = self.build_discriminator() self.d_B = self.build_discriminator() self.d_A.compile(loss='mse', optimizer=optimizer, metrics=['accuracy']) self.d_B.compile(loss='mse', optimizer=optimizer, metrics=['accuracy']) #------------------------- # Construct Computational # Graph of Generators #------------------------- # Build the generators self.g_AB = self.build_generator() self.g_BA = self.build_generator() # Input images from both domains img_A = Input(shape=self.img_shape) img_B = Input(shape=self.img_shape) # Translate images to the other domain fake_B = self.g_AB(img_A) fake_A = self.g_BA(img_B) # Translate images back to original domain reconstr_A = self.g_BA(fake_B) reconstr_B = self.g_AB(fake_A) # Identity mapping of images img_A_id = self.g_BA(img_A) img_B_id = self.g_AB(img_B) # For the combined model we will only train the generators self.d_A.trainable = False self.d_B.trainable = False # Discriminators determines validity of translated images valid_A = self.d_A(fake_A) valid_B = self.d_B(fake_B) # Combined model trains generators to fool discriminators self.combined = Model(inputs=[img_A, img_B], outputs=[ valid_A, valid_B, reconstr_A, reconstr_B, img_A_id, img_B_id ]) self.combined.compile(loss=['mse', 'mse', 'mae', 'mae', 'mae', 'mae'], loss_weights=[ 1, 1, self.lambda_cycle, self.lambda_cycle, self.lambda_id, self.lambda_id ], optimizer=optimizer) def build_generator(self): """U-Net Generator""" def conv2d(layer_input, filters, f_size=4): """Layers used during downsampling""" d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input) d = LeakyReLU(alpha=0.2)(d) d = InstanceNormalization()(d) return d def deconv2d(layer_input, skip_input, filters, f_size=4, dropout_rate=0): """Layers used during upsampling""" u = UpSampling2D(size=2)(layer_input) u = Conv2D(filters, kernel_size=f_size, strides=1, padding='same', activation='relu')(u) if dropout_rate: u = Dropout(dropout_rate)(u) u = InstanceNormalization()(u) u = Concatenate()([u, skip_input]) return u # Image input d0 = Input(shape=self.img_shape) # Downsampling d1 = conv2d(d0, self.gf) d2 = conv2d(d1, self.gf*2) d3 = conv2d(d2, self.gf*4) d4 = conv2d(d3, self.gf*8) # Upsampling u1 = deconv2d(d4, d3, self.gf*4) u2 = deconv2d(u1, d2, self.gf*2) u3 = deconv2d(u2, d1, self.gf) u4 = UpSampling2D(size=2)(u3) output_img = Conv2D(self.channels, kernel_size=4, strides=1, padding='same', activation='tanh')(u4) return Model(d0, output_img) def build_discriminator(self): def d_layer(layer_input, filters, f_size=4, normalization=True): """Discriminator layer""" d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input) d = LeakyReLU(alpha=0.2)(d) if normalization: d = InstanceNormalization()(d) return d img = Input(shape=self.img_shape) d1 = d_layer(img, self.df, normalization=False) d2 = d_layer(d1, self.df*2) d3 = d_layer(d2, self.df*4) d4 = d_layer(d3, self.df*8) validity = Conv2D(1, kernel_size=4, strides=1, padding='same')(d4) return Model(img, validity) def train(self, epochs, batch_size=1, sample_interval=50): start_time = datetime.datetime.now() # Adversarial loss ground truths valid = np.ones((batch_size,) + self.disc_patch) fake = np.zeros((batch_size,) + self.disc_patch) for epoch in range(epochs): for batch_i, (imgs_A, imgs_B) in enumerate(self.data_loader.load_batch(batch_size)): # ---------------------- # Train Discriminators # ---------------------- # Translate images to opposite domain fake_B = self.g_AB.predict(imgs_A) fake_A = self.g_BA.predict(imgs_B) # Train the discriminators (original images = real / translated = Fake) dA_loss_real = self.d_A.train_on_batch(imgs_A, valid) dA_loss_fake = self.d_A.train_on_batch(fake_A, fake) dA_loss = 0.5 * np.add(dA_loss_real, dA_loss_fake) dB_loss_real = self.d_B.train_on_batch(imgs_B, valid) dB_loss_fake = self.d_B.train_on_batch(fake_B, fake) dB_loss = 0.5 * np.add(dB_loss_real, dB_loss_fake) # Total disciminator loss d_loss = 0.5 * np.add(dA_loss, dB_loss) # ------------------ # Train Generators # ------------------ # Train the generators g_loss = self.combined.train_on_batch([imgs_A, imgs_B], [valid, valid, imgs_A, imgs_B, imgs_A, imgs_B]) elapsed_time = datetime.datetime.now() - start_time # Plot the progress print ("[Epoch %d/%d] [Batch %d/%d] [D loss: %f, acc: %3d%%] [G loss: %05f, adv: %05f, recon: %05f, id: %05f] time: %s " \ % ( epoch, epochs, batch_i, self.data_loader.n_batches, d_loss[0], 100*d_loss[1], g_loss[0], np.mean(g_loss[1:3]), np.mean(g_loss[3:5]), np.mean(g_loss[5:6]), elapsed_time)) # If at save interval => save generated image samples if batch_i % sample_interval == 0: self.sample_images(epoch, batch_i) def sample_images(self, epoch, batch_i): os.makedirs('images/%s' % self.dataset_name, exist_ok=True) r, c = 2, 3 imgs_A = self.data_loader.load_data(domain="A", batch_size=1, is_testing=True) imgs_B = self.data_loader.load_data(domain="B", batch_size=1, is_testing=True) # Demo (for GIF) #imgs_A = self.data_loader.load_img('datasets/apple2orange/testA/n07740461_1541.jpg') #imgs_B = self.data_loader.load_img('datasets/apple2orange/testB/n07749192_4241.jpg') # Translate images to the other domain fake_B = self.g_AB.predict(imgs_A) fake_A = self.g_BA.predict(imgs_B) # Translate back to original domain reconstr_A = self.g_BA.predict(fake_B) reconstr_B = self.g_AB.predict(fake_A) gen_imgs = np.concatenate([imgs_A, fake_B, reconstr_A, imgs_B, fake_A, reconstr_B]) # Rescale images 0 - 1 gen_imgs = 0.5 * gen_imgs + 0.5 titles = ['Original', 'Translated', 'Reconstructed'] fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): axs[i,j].imshow(gen_imgs[cnt]) axs[i, j].set_title(titles[j]) axs[i,j].axis('off') cnt += 1 fig.savefig("images/%s/%d_%d.png" % (self.dataset_name, epoch, batch_i)) plt.close() if __name__ == '__main__': gan = CycleGAN() gan.train(epochs=200, batch_size=1, sample_interval=200) ================================================ FILE: cyclegan/data_loader.py ================================================ import scipy from glob import glob import numpy as np class DataLoader(): def __init__(self, dataset_name, img_res=(128, 128)): self.dataset_name = dataset_name self.img_res = img_res def load_data(self, domain, batch_size=1, is_testing=False): data_type = "train%s" % domain if not is_testing else "test%s" % domain path = glob('./datasets/%s/%s/*' % (self.dataset_name, data_type)) batch_images = np.random.choice(path, size=batch_size) imgs = [] for img_path in batch_images: img = self.imread(img_path) if not is_testing: img = scipy.misc.imresize(img, self.img_res) if np.random.random() > 0.5: img = np.fliplr(img) else: img = scipy.misc.imresize(img, self.img_res) imgs.append(img) imgs = np.array(imgs)/127.5 - 1. return imgs def load_batch(self, batch_size=1, is_testing=False): data_type = "train" if not is_testing else "val" path_A = glob('./datasets/%s/%sA/*' % (self.dataset_name, data_type)) path_B = glob('./datasets/%s/%sB/*' % (self.dataset_name, data_type)) self.n_batches = int(min(len(path_A), len(path_B)) / batch_size) total_samples = self.n_batches * batch_size # Sample n_batches * batch_size from each path list so that model sees all # samples from both domains path_A = np.random.choice(path_A, total_samples, replace=False) path_B = np.random.choice(path_B, total_samples, replace=False) for i in range(self.n_batches-1): batch_A = path_A[i*batch_size:(i+1)*batch_size] batch_B = path_B[i*batch_size:(i+1)*batch_size] imgs_A, imgs_B = [], [] for img_A, img_B in zip(batch_A, batch_B): img_A = self.imread(img_A) img_B = self.imread(img_B) img_A = scipy.misc.imresize(img_A, self.img_res) img_B = scipy.misc.imresize(img_B, self.img_res) if not is_testing and np.random.random() > 0.5: img_A = np.fliplr(img_A) img_B = np.fliplr(img_B) imgs_A.append(img_A) imgs_B.append(img_B) imgs_A = np.array(imgs_A)/127.5 - 1. imgs_B = np.array(imgs_B)/127.5 - 1. yield imgs_A, imgs_B def load_img(self, path): img = self.imread(path) img = scipy.misc.imresize(img, self.img_res) img = img/127.5 - 1. return img[np.newaxis, :, :, :] def imread(self, path): return scipy.misc.imread(path, mode='RGB').astype(np.float) ================================================ FILE: cyclegan/download_dataset.sh ================================================ mkdir datasets FILE=$1 if [[ $FILE != "ae_photos" && $FILE != "apple2orange" && $FILE != "summer2winter_yosemite" && $FILE != "horse2zebra" && $FILE != "monet2photo" && $FILE != "cezanne2photo" && $FILE != "ukiyoe2photo" && $FILE != "vangogh2photo" && $FILE != "maps" && $FILE != "cityscapes" && $FILE != "facades" && $FILE != "iphone2dslr_flower" && $FILE != "ae_photos" ]]; then echo "Available datasets are: apple2orange, summer2winter_yosemite, horse2zebra, monet2photo, cezanne2photo, ukiyoe2photo, vangogh2photo, maps, cityscapes, facades, iphone2dslr_flower, ae_photos" exit 1 fi URL=https://people.eecs.berkeley.edu/~taesung_park/CycleGAN/datasets/$FILE.zip ZIP_FILE=./datasets/$FILE.zip TARGET_DIR=./datasets/$FILE/ wget -N $URL -O $ZIP_FILE mkdir $TARGET_DIR unzip $ZIP_FILE -d ./datasets/ rm $ZIP_FILE ================================================ FILE: cyclegan/images/.gitignore ================================================ * !.gitignore ================================================ FILE: cyclegan/saved_model/.gitignore ================================================ * !.gitignore ================================================ FILE: dcgan/dcgan.py ================================================ from __future__ import print_function, division from keras.datasets import mnist from keras.layers import Input, Dense, Reshape, Flatten, Dropout from keras.layers import BatchNormalization, Activation, ZeroPadding2D from keras.layers.advanced_activations import LeakyReLU from keras.layers.convolutional import UpSampling2D, Conv2D from keras.models import Sequential, Model from keras.optimizers import Adam import matplotlib.pyplot as plt import sys import numpy as np class DCGAN(): def __init__(self): # Input shape self.img_rows = 28 self.img_cols = 28 self.channels = 1 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.latent_dim = 100 optimizer = Adam(0.0002, 0.5) # Build and compile the discriminator self.discriminator = self.build_discriminator() self.discriminator.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) # Build the generator self.generator = self.build_generator() # The generator takes noise as input and generates imgs z = Input(shape=(self.latent_dim,)) img = self.generator(z) # For the combined model we will only train the generator self.discriminator.trainable = False # The discriminator takes generated images as input and determines validity valid = self.discriminator(img) # The combined model (stacked generator and discriminator) # Trains the generator to fool the discriminator self.combined = Model(z, valid) self.combined.compile(loss='binary_crossentropy', optimizer=optimizer) def build_generator(self): model = Sequential() model.add(Dense(128 * 7 * 7, activation="relu", input_dim=self.latent_dim)) model.add(Reshape((7, 7, 128))) model.add(UpSampling2D()) model.add(Conv2D(128, kernel_size=3, padding="same")) model.add(BatchNormalization(momentum=0.8)) model.add(Activation("relu")) model.add(UpSampling2D()) model.add(Conv2D(64, kernel_size=3, padding="same")) model.add(BatchNormalization(momentum=0.8)) model.add(Activation("relu")) model.add(Conv2D(self.channels, kernel_size=3, padding="same")) model.add(Activation("tanh")) model.summary() noise = Input(shape=(self.latent_dim,)) img = model(noise) return Model(noise, img) def build_discriminator(self): model = Sequential() model.add(Conv2D(32, kernel_size=3, strides=2, input_shape=self.img_shape, padding="same")) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Conv2D(64, kernel_size=3, strides=2, padding="same")) model.add(ZeroPadding2D(padding=((0,1),(0,1)))) model.add(BatchNormalization(momentum=0.8)) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Conv2D(128, kernel_size=3, strides=2, padding="same")) model.add(BatchNormalization(momentum=0.8)) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Conv2D(256, kernel_size=3, strides=1, padding="same")) model.add(BatchNormalization(momentum=0.8)) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(1, activation='sigmoid')) model.summary() img = Input(shape=self.img_shape) validity = model(img) return Model(img, validity) def train(self, epochs, batch_size=128, save_interval=50): # Load the dataset (X_train, _), (_, _) = mnist.load_data() # Rescale -1 to 1 X_train = X_train / 127.5 - 1. X_train = np.expand_dims(X_train, axis=3) # Adversarial ground truths valid = np.ones((batch_size, 1)) fake = np.zeros((batch_size, 1)) for epoch in range(epochs): # --------------------- # Train Discriminator # --------------------- # Select a random half of images idx = np.random.randint(0, X_train.shape[0], batch_size) imgs = X_train[idx] # Sample noise and generate a batch of new images noise = np.random.normal(0, 1, (batch_size, self.latent_dim)) gen_imgs = self.generator.predict(noise) # Train the discriminator (real classified as ones and generated as zeros) d_loss_real = self.discriminator.train_on_batch(imgs, valid) d_loss_fake = self.discriminator.train_on_batch(gen_imgs, fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # --------------------- # Train Generator # --------------------- # Train the generator (wants discriminator to mistake images as real) g_loss = self.combined.train_on_batch(noise, valid) # Plot the progress print ("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]" % (epoch, d_loss[0], 100*d_loss[1], g_loss)) # If at save interval => save generated image samples if epoch % save_interval == 0: self.save_imgs(epoch) def save_imgs(self, epoch): r, c = 5, 5 noise = np.random.normal(0, 1, (r * c, self.latent_dim)) gen_imgs = self.generator.predict(noise) # Rescale images 0 - 1 gen_imgs = 0.5 * gen_imgs + 0.5 fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): axs[i,j].imshow(gen_imgs[cnt, :,:,0], cmap='gray') axs[i,j].axis('off') cnt += 1 fig.savefig("images/mnist_%d.png" % epoch) plt.close() if __name__ == '__main__': dcgan = DCGAN() dcgan.train(epochs=4000, batch_size=32, save_interval=50) ================================================ FILE: dcgan/images/.gitignore ================================================ * !.gitignore ================================================ FILE: dcgan/saved_model/.gitignore ================================================ * !.gitignore ================================================ FILE: discogan/data_loader.py ================================================ import scipy from glob import glob import numpy as np class DataLoader(): def __init__(self, dataset_name, img_res=(128, 128)): self.dataset_name = dataset_name self.img_res = img_res def load_data(self, batch_size=1, is_testing=False): data_type = "train" if not is_testing else "val" path = glob('./datasets/%s/%s/*' % (self.dataset_name, data_type)) batch = np.random.choice(path, size=batch_size) imgs_A, imgs_B = [], [] for img in batch: img = self.imread(img) h, w, _ = img.shape half_w = int(w/2) img_A = img[:, :half_w, :] img_B = img[:, half_w:, :] img_A = scipy.misc.imresize(img_A, self.img_res) img_B = scipy.misc.imresize(img_B, self.img_res) if not is_testing and np.random.random() > 0.5: img_A = np.fliplr(img_A) img_B = np.fliplr(img_B) imgs_A.append(img_A) imgs_B.append(img_B) imgs_A = np.array(imgs_A)/127.5 - 1. imgs_B = np.array(imgs_B)/127.5 - 1. return imgs_A, imgs_B def load_batch(self, batch_size=1, is_testing=False): data_type = "train" if not is_testing else "val" path = glob('./datasets/%s/%s/*' % (self.dataset_name, data_type)) self.n_batches = int(len(path) / batch_size) for i in range(self.n_batches-1): batch = path[i*batch_size:(i+1)*batch_size] imgs_A, imgs_B = [], [] for img in batch: img = self.imread(img) h, w, _ = img.shape half_w = int(w/2) img_A = img[:, :half_w, :] img_B = img[:, half_w:, :] img_A = scipy.misc.imresize(img_A, self.img_res) img_B = scipy.misc.imresize(img_B, self.img_res) if not is_testing and np.random.random() > 0.5: img_A = np.fliplr(img_A) img_B = np.fliplr(img_B) imgs_A.append(img_A) imgs_B.append(img_B) imgs_A = np.array(imgs_A)/127.5 - 1. imgs_B = np.array(imgs_B)/127.5 - 1. yield imgs_A, imgs_B def load_img(self, path): img = self.imread(path) img = scipy.misc.imresize(img, self.img_res) img = img/127.5 - 1. return img[np.newaxis, :, :, :] def imread(self, path): return scipy.misc.imread(path, mode='RGB').astype(np.float) ================================================ FILE: discogan/discogan.py ================================================ from __future__ import print_function, division import scipy from keras.datasets import mnist from keras_contrib.layers.normalization.instancenormalization import InstanceNormalization from keras.layers import Input, Dense, Reshape, Flatten, Dropout, Concatenate from keras.layers import BatchNormalization, Activation, ZeroPadding2D from keras.layers.advanced_activations import LeakyReLU from keras.layers.convolutional import UpSampling2D, Conv2D from keras.models import Sequential, Model from keras.optimizers import Adam import datetime import matplotlib.pyplot as plt import sys from data_loader import DataLoader import numpy as np import os class DiscoGAN(): def __init__(self): # Input shape self.img_rows = 128 self.img_cols = 128 self.channels = 3 self.img_shape = (self.img_rows, self.img_cols, self.channels) # Configure data loader self.dataset_name = 'edges2shoes' self.data_loader = DataLoader(dataset_name=self.dataset_name, img_res=(self.img_rows, self.img_cols)) # Calculate output shape of D (PatchGAN) patch = int(self.img_rows / 2**4) self.disc_patch = (patch, patch, 1) # Number of filters in the first layer of G and D self.gf = 64 self.df = 64 optimizer = Adam(0.0002, 0.5) # Build and compile the discriminators self.d_A = self.build_discriminator() self.d_B = self.build_discriminator() self.d_A.compile(loss='mse', optimizer=optimizer, metrics=['accuracy']) self.d_B.compile(loss='mse', optimizer=optimizer, metrics=['accuracy']) #------------------------- # Construct Computational # Graph of Generators #------------------------- # Build the generators self.g_AB = self.build_generator() self.g_BA = self.build_generator() # Input images from both domains img_A = Input(shape=self.img_shape) img_B = Input(shape=self.img_shape) # Translate images to the other domain fake_B = self.g_AB(img_A) fake_A = self.g_BA(img_B) # Translate images back to original domain reconstr_A = self.g_BA(fake_B) reconstr_B = self.g_AB(fake_A) # For the combined model we will only train the generators self.d_A.trainable = False self.d_B.trainable = False # Discriminators determines validity of translated images valid_A = self.d_A(fake_A) valid_B = self.d_B(fake_B) # Objectives # + Adversarial: Fool domain discriminators # + Translation: Minimize MAE between e.g. fake B and true B # + Cycle-consistency: Minimize MAE between reconstructed images and original self.combined = Model(inputs=[img_A, img_B], outputs=[ valid_A, valid_B, fake_B, fake_A, reconstr_A, reconstr_B ]) self.combined.compile(loss=['mse', 'mse', 'mae', 'mae', 'mae', 'mae'], optimizer=optimizer) def build_generator(self): """U-Net Generator""" def conv2d(layer_input, filters, f_size=4, normalize=True): """Layers used during downsampling""" d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input) d = LeakyReLU(alpha=0.2)(d) if normalize: d = InstanceNormalization()(d) return d def deconv2d(layer_input, skip_input, filters, f_size=4, dropout_rate=0): """Layers used during upsampling""" u = UpSampling2D(size=2)(layer_input) u = Conv2D(filters, kernel_size=f_size, strides=1, padding='same', activation='relu')(u) if dropout_rate: u = Dropout(dropout_rate)(u) u = InstanceNormalization()(u) u = Concatenate()([u, skip_input]) return u # Image input d0 = Input(shape=self.img_shape) # Downsampling d1 = conv2d(d0, self.gf, normalize=False) d2 = conv2d(d1, self.gf*2) d3 = conv2d(d2, self.gf*4) d4 = conv2d(d3, self.gf*8) d5 = conv2d(d4, self.gf*8) d6 = conv2d(d5, self.gf*8) d7 = conv2d(d6, self.gf*8) # Upsampling u1 = deconv2d(d7, d6, self.gf*8) u2 = deconv2d(u1, d5, self.gf*8) u3 = deconv2d(u2, d4, self.gf*8) u4 = deconv2d(u3, d3, self.gf*4) u5 = deconv2d(u4, d2, self.gf*2) u6 = deconv2d(u5, d1, self.gf) u7 = UpSampling2D(size=2)(u6) output_img = Conv2D(self.channels, kernel_size=4, strides=1, padding='same', activation='tanh')(u7) return Model(d0, output_img) def build_discriminator(self): def d_layer(layer_input, filters, f_size=4, normalization=True): """Discriminator layer""" d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input) d = LeakyReLU(alpha=0.2)(d) if normalization: d = InstanceNormalization()(d) return d img = Input(shape=self.img_shape) d1 = d_layer(img, self.df, normalization=False) d2 = d_layer(d1, self.df*2) d3 = d_layer(d2, self.df*4) d4 = d_layer(d3, self.df*8) validity = Conv2D(1, kernel_size=4, strides=1, padding='same')(d4) return Model(img, validity) def train(self, epochs, batch_size=128, sample_interval=50): start_time = datetime.datetime.now() # Adversarial loss ground truths valid = np.ones((batch_size,) + self.disc_patch) fake = np.zeros((batch_size,) + self.disc_patch) for epoch in range(epochs): for batch_i, (imgs_A, imgs_B) in enumerate(self.data_loader.load_batch(batch_size)): # ---------------------- # Train Discriminators # ---------------------- # Translate images to opposite domain fake_B = self.g_AB.predict(imgs_A) fake_A = self.g_BA.predict(imgs_B) # Train the discriminators (original images = real / translated = Fake) dA_loss_real = self.d_A.train_on_batch(imgs_A, valid) dA_loss_fake = self.d_A.train_on_batch(fake_A, fake) dA_loss = 0.5 * np.add(dA_loss_real, dA_loss_fake) dB_loss_real = self.d_B.train_on_batch(imgs_B, valid) dB_loss_fake = self.d_B.train_on_batch(fake_B, fake) dB_loss = 0.5 * np.add(dB_loss_real, dB_loss_fake) # Total disciminator loss d_loss = 0.5 * np.add(dA_loss, dB_loss) # ------------------ # Train Generators # ------------------ # Train the generators g_loss = self.combined.train_on_batch([imgs_A, imgs_B], [valid, valid, \ imgs_B, imgs_A, \ imgs_A, imgs_B]) elapsed_time = datetime.datetime.now() - start_time # Plot the progress print ("[%d] [%d/%d] time: %s, [d_loss: %f, g_loss: %f]" % (epoch, batch_i, self.data_loader.n_batches, elapsed_time, d_loss[0], g_loss[0])) # If at save interval => save generated image samples if batch_i % sample_interval == 0: self.sample_images(epoch, batch_i) def sample_images(self, epoch, batch_i): os.makedirs('images/%s' % self.dataset_name, exist_ok=True) r, c = 2, 3 imgs_A, imgs_B = self.data_loader.load_data(batch_size=1, is_testing=True) # Translate images to the other domain fake_B = self.g_AB.predict(imgs_A) fake_A = self.g_BA.predict(imgs_B) # Translate back to original domain reconstr_A = self.g_BA.predict(fake_B) reconstr_B = self.g_AB.predict(fake_A) gen_imgs = np.concatenate([imgs_A, fake_B, reconstr_A, imgs_B, fake_A, reconstr_B]) # Rescale images 0 - 1 gen_imgs = 0.5 * gen_imgs + 0.5 titles = ['Original', 'Translated', 'Reconstructed'] fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): axs[i,j].imshow(gen_imgs[cnt]) axs[i, j].set_title(titles[j]) axs[i,j].axis('off') cnt += 1 fig.savefig("images/%s/%d_%d.png" % (self.dataset_name, epoch, batch_i)) plt.close() if __name__ == '__main__': gan = DiscoGAN() gan.train(epochs=20, batch_size=1, sample_interval=200) ================================================ FILE: discogan/download_dataset.sh ================================================ mkdir datasets FILE=$1 URL=https://people.eecs.berkeley.edu/~tinghuiz/projects/pix2pix/datasets/$FILE.tar.gz TAR_FILE=./datasets/$FILE.tar.gz TARGET_DIR=./datasets/$FILE/ wget -N $URL -O $TAR_FILE mkdir $TARGET_DIR tar -zxvf $TAR_FILE -C ./datasets/ rm $TAR_FILE ================================================ FILE: discogan/images/.gitignore ================================================ * !.gitignore ================================================ FILE: discogan/saved_model/.gitignore ================================================ * !.gitignore ================================================ FILE: dualgan/dualgan.py ================================================ from __future__ import print_function, division import scipy from keras.datasets import mnist from keras.layers import Input, Dense, Reshape, Flatten, Dropout, Concatenate from keras.layers import BatchNormalization, Activation, ZeroPadding2D from keras.layers.advanced_activations import LeakyReLU from keras.layers.convolutional import UpSampling2D, Conv2D from keras.models import Sequential, Model from keras.optimizers import RMSprop, Adam from keras.utils import to_categorical import keras.backend as K import matplotlib.pyplot as plt import sys import numpy as np class DUALGAN(): def __init__(self): self.img_rows = 28 self.img_cols = 28 self.channels = 1 self.img_dim = self.img_rows*self.img_cols optimizer = Adam(0.0002, 0.5) # Build and compile the discriminators self.D_A = self.build_discriminator() self.D_A.compile(loss=self.wasserstein_loss, optimizer=optimizer, metrics=['accuracy']) self.D_B = self.build_discriminator() self.D_B.compile(loss=self.wasserstein_loss, optimizer=optimizer, metrics=['accuracy']) #------------------------- # Construct Computational # Graph of Generators #------------------------- # Build the generators self.G_AB = self.build_generator() self.G_BA = self.build_generator() # For the combined model we will only train the generators self.D_A.trainable = False self.D_B.trainable = False # The generator takes images from their respective domains as inputs imgs_A = Input(shape=(self.img_dim,)) imgs_B = Input(shape=(self.img_dim,)) # Generators translates the images to the opposite domain fake_B = self.G_AB(imgs_A) fake_A = self.G_BA(imgs_B) # The discriminators determines validity of translated images valid_A = self.D_A(fake_A) valid_B = self.D_B(fake_B) # Generators translate the images back to their original domain recov_A = self.G_BA(fake_B) recov_B = self.G_AB(fake_A) # The combined model (stacked generators and discriminators) self.combined = Model(inputs=[imgs_A, imgs_B], outputs=[valid_A, valid_B, recov_A, recov_B]) self.combined.compile(loss=[self.wasserstein_loss, self.wasserstein_loss, 'mae', 'mae'], optimizer=optimizer, loss_weights=[1, 1, 100, 100]) def build_generator(self): X = Input(shape=(self.img_dim,)) model = Sequential() model.add(Dense(256, input_dim=self.img_dim)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dropout(0.4)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dropout(0.4)) model.add(Dense(1024)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dropout(0.4)) model.add(Dense(self.img_dim, activation='tanh')) X_translated = model(X) return Model(X, X_translated) def build_discriminator(self): img = Input(shape=(self.img_dim,)) model = Sequential() model.add(Dense(512, input_dim=self.img_dim)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(256)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(1)) validity = model(img) return Model(img, validity) def sample_generator_input(self, X, batch_size): # Sample random batch of images from X idx = np.random.randint(0, X.shape[0], batch_size) return X[idx] def wasserstein_loss(self, y_true, y_pred): return K.mean(y_true * y_pred) def train(self, epochs, batch_size=128, sample_interval=50): # Load the dataset (X_train, _), (_, _) = mnist.load_data() # Rescale -1 to 1 X_train = (X_train.astype(np.float32) - 127.5) / 127.5 # Domain A and B (rotated) X_A = X_train[:int(X_train.shape[0]/2)] X_B = scipy.ndimage.interpolation.rotate(X_train[int(X_train.shape[0]/2):], 90, axes=(1, 2)) X_A = X_A.reshape(X_A.shape[0], self.img_dim) X_B = X_B.reshape(X_B.shape[0], self.img_dim) clip_value = 0.01 n_critic = 4 # Adversarial ground truths valid = -np.ones((batch_size, 1)) fake = np.ones((batch_size, 1)) for epoch in range(epochs): # Train the discriminator for n_critic iterations for _ in range(n_critic): # ---------------------- # Train Discriminators # ---------------------- # Sample generator inputs imgs_A = self.sample_generator_input(X_A, batch_size) imgs_B = self.sample_generator_input(X_B, batch_size) # Translate images to their opposite domain fake_B = self.G_AB.predict(imgs_A) fake_A = self.G_BA.predict(imgs_B) # Train the discriminators D_A_loss_real = self.D_A.train_on_batch(imgs_A, valid) D_A_loss_fake = self.D_A.train_on_batch(fake_A, fake) D_B_loss_real = self.D_B.train_on_batch(imgs_B, valid) D_B_loss_fake = self.D_B.train_on_batch(fake_B, fake) D_A_loss = 0.5 * np.add(D_A_loss_real, D_A_loss_fake) D_B_loss = 0.5 * np.add(D_B_loss_real, D_B_loss_fake) # Clip discriminator weights for d in [self.D_A, self.D_B]: for l in d.layers: weights = l.get_weights() weights = [np.clip(w, -clip_value, clip_value) for w in weights] l.set_weights(weights) # ------------------ # Train Generators # ------------------ # Train the generators g_loss = self.combined.train_on_batch([imgs_A, imgs_B], [valid, valid, imgs_A, imgs_B]) # Plot the progress print ("%d [D1 loss: %f] [D2 loss: %f] [G loss: %f]" \ % (epoch, D_A_loss[0], D_B_loss[0], g_loss[0])) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.save_imgs(epoch, X_A, X_B) def save_imgs(self, epoch, X_A, X_B): r, c = 4, 4 # Sample generator inputs imgs_A = self.sample_generator_input(X_A, c) imgs_B = self.sample_generator_input(X_B, c) # Images translated to their opposite domain fake_B = self.G_AB.predict(imgs_A) fake_A = self.G_BA.predict(imgs_B) gen_imgs = np.concatenate([imgs_A, fake_B, imgs_B, fake_A]) gen_imgs = gen_imgs.reshape((r, c, self.img_rows, self.img_cols, 1)) # Rescale images 0 - 1 gen_imgs = 0.5 * gen_imgs + 0.5 fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): axs[i,j].imshow(gen_imgs[i, j, :,:,0], cmap='gray') axs[i,j].axis('off') cnt += 1 fig.savefig("images/mnist_%d.png" % epoch) plt.close() if __name__ == '__main__': gan = DUALGAN() gan.train(epochs=30000, batch_size=32, sample_interval=200) ================================================ FILE: dualgan/images/.gitignore ================================================ * !.gitignore ================================================ FILE: dualgan/saved_model/.gitignore ================================================ * !.gitignore ================================================ FILE: gan/gan.py ================================================ from __future__ import print_function, division from keras.datasets import mnist from keras.layers import Input, Dense, Reshape, Flatten, Dropout from keras.layers import BatchNormalization, Activation, ZeroPadding2D from keras.layers.advanced_activations import LeakyReLU from keras.layers.convolutional import UpSampling2D, Conv2D from keras.models import Sequential, Model from keras.optimizers import Adam import matplotlib.pyplot as plt import sys import numpy as np class GAN(): def __init__(self): self.img_rows = 28 self.img_cols = 28 self.channels = 1 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.latent_dim = 100 optimizer = Adam(0.0002, 0.5) # Build and compile the discriminator self.discriminator = self.build_discriminator() self.discriminator.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) # Build the generator self.generator = self.build_generator() # The generator takes noise as input and generates imgs z = Input(shape=(self.latent_dim,)) img = self.generator(z) # For the combined model we will only train the generator self.discriminator.trainable = False # The discriminator takes generated images as input and determines validity validity = self.discriminator(img) # The combined model (stacked generator and discriminator) # Trains the generator to fool the discriminator self.combined = Model(z, validity) self.combined.compile(loss='binary_crossentropy', optimizer=optimizer) def build_generator(self): model = Sequential() model.add(Dense(256, input_dim=self.latent_dim)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(1024)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(np.prod(self.img_shape), activation='tanh')) model.add(Reshape(self.img_shape)) model.summary() noise = Input(shape=(self.latent_dim,)) img = model(noise) return Model(noise, img) def build_discriminator(self): model = Sequential() model.add(Flatten(input_shape=self.img_shape)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(256)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(1, activation='sigmoid')) model.summary() img = Input(shape=self.img_shape) validity = model(img) return Model(img, validity) def train(self, epochs, batch_size=128, sample_interval=50): # Load the dataset (X_train, _), (_, _) = mnist.load_data() # Rescale -1 to 1 X_train = X_train / 127.5 - 1. X_train = np.expand_dims(X_train, axis=3) # Adversarial ground truths valid = np.ones((batch_size, 1)) fake = np.zeros((batch_size, 1)) for epoch in range(epochs): # --------------------- # Train Discriminator # --------------------- # Select a random batch of images idx = np.random.randint(0, X_train.shape[0], batch_size) imgs = X_train[idx] noise = np.random.normal(0, 1, (batch_size, self.latent_dim)) # Generate a batch of new images gen_imgs = self.generator.predict(noise) # Train the discriminator d_loss_real = self.discriminator.train_on_batch(imgs, valid) d_loss_fake = self.discriminator.train_on_batch(gen_imgs, fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # --------------------- # Train Generator # --------------------- noise = np.random.normal(0, 1, (batch_size, self.latent_dim)) # Train the generator (to have the discriminator label samples as valid) g_loss = self.combined.train_on_batch(noise, valid) # Plot the progress print ("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]" % (epoch, d_loss[0], 100*d_loss[1], g_loss)) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.sample_images(epoch) def sample_images(self, epoch): r, c = 5, 5 noise = np.random.normal(0, 1, (r * c, self.latent_dim)) gen_imgs = self.generator.predict(noise) # Rescale images 0 - 1 gen_imgs = 0.5 * gen_imgs + 0.5 fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): axs[i,j].imshow(gen_imgs[cnt, :,:,0], cmap='gray') axs[i,j].axis('off') cnt += 1 fig.savefig("images/%d.png" % epoch) plt.close() if __name__ == '__main__': gan = GAN() gan.train(epochs=30000, batch_size=32, sample_interval=200) ================================================ FILE: gan/images/.gitignore ================================================ * !.gitignore ================================================ FILE: gan/saved_model/.gitignore ================================================ * !.gitignore ================================================ FILE: infogan/images/.gitignore ================================================ * !.gitignore ================================================ FILE: infogan/infogan.py ================================================ from __future__ import print_function, division from keras.datasets import mnist from keras.layers import Input, Dense, Reshape, Flatten, Dropout, multiply, concatenate from keras.layers import BatchNormalization, Activation, Embedding, ZeroPadding2D, Lambda from keras.layers.advanced_activations import LeakyReLU from keras.layers.convolutional import UpSampling2D, Conv2D from keras.models import Sequential, Model from keras.optimizers import Adam from keras.utils import to_categorical import keras.backend as K import matplotlib.pyplot as plt import numpy as np class INFOGAN(): def __init__(self): self.img_rows = 28 self.img_cols = 28 self.channels = 1 self.num_classes = 10 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.latent_dim = 72 optimizer = Adam(0.0002, 0.5) losses = ['binary_crossentropy', self.mutual_info_loss] # Build and the discriminator and recognition network self.discriminator, self.auxilliary = self.build_disk_and_q_net() self.discriminator.compile(loss=['binary_crossentropy'], optimizer=optimizer, metrics=['accuracy']) # Build and compile the recognition network Q self.auxilliary.compile(loss=[self.mutual_info_loss], optimizer=optimizer, metrics=['accuracy']) # Build the generator self.generator = self.build_generator() # The generator takes noise and the target label as input # and generates the corresponding digit of that label gen_input = Input(shape=(self.latent_dim,)) img = self.generator(gen_input) # For the combined model we will only train the generator self.discriminator.trainable = False # The discriminator takes generated image as input and determines validity valid = self.discriminator(img) # The recognition network produces the label target_label = self.auxilliary(img) # The combined model (stacked generator and discriminator) self.combined = Model(gen_input, [valid, target_label]) self.combined.compile(loss=losses, optimizer=optimizer) def build_generator(self): model = Sequential() model.add(Dense(128 * 7 * 7, activation="relu", input_dim=self.latent_dim)) model.add(Reshape((7, 7, 128))) model.add(BatchNormalization(momentum=0.8)) model.add(UpSampling2D()) model.add(Conv2D(128, kernel_size=3, padding="same")) model.add(Activation("relu")) model.add(BatchNormalization(momentum=0.8)) model.add(UpSampling2D()) model.add(Conv2D(64, kernel_size=3, padding="same")) model.add(Activation("relu")) model.add(BatchNormalization(momentum=0.8)) model.add(Conv2D(self.channels, kernel_size=3, padding='same')) model.add(Activation("tanh")) gen_input = Input(shape=(self.latent_dim,)) img = model(gen_input) model.summary() return Model(gen_input, img) def build_disk_and_q_net(self): img = Input(shape=self.img_shape) # Shared layers between discriminator and recognition network model = Sequential() model.add(Conv2D(64, kernel_size=3, strides=2, input_shape=self.img_shape, padding="same")) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Conv2D(128, kernel_size=3, strides=2, padding="same")) model.add(ZeroPadding2D(padding=((0,1),(0,1)))) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(BatchNormalization(momentum=0.8)) model.add(Conv2D(256, kernel_size=3, strides=2, padding="same")) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(BatchNormalization(momentum=0.8)) model.add(Conv2D(512, kernel_size=3, strides=2, padding="same")) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(BatchNormalization(momentum=0.8)) model.add(Flatten()) img_embedding = model(img) # Discriminator validity = Dense(1, activation='sigmoid')(img_embedding) # Recognition q_net = Dense(128, activation='relu')(img_embedding) label = Dense(self.num_classes, activation='softmax')(q_net) # Return discriminator and recognition network return Model(img, validity), Model(img, label) def mutual_info_loss(self, c, c_given_x): """The mutual information metric we aim to minimize""" eps = 1e-8 conditional_entropy = K.mean(- K.sum(K.log(c_given_x + eps) * c, axis=1)) entropy = K.mean(- K.sum(K.log(c + eps) * c, axis=1)) return conditional_entropy + entropy def sample_generator_input(self, batch_size): # Generator inputs sampled_noise = np.random.normal(0, 1, (batch_size, 62)) sampled_labels = np.random.randint(0, self.num_classes, batch_size).reshape(-1, 1) sampled_labels = to_categorical(sampled_labels, num_classes=self.num_classes) return sampled_noise, sampled_labels def train(self, epochs, batch_size=128, sample_interval=50): # Load the dataset (X_train, y_train), (_, _) = mnist.load_data() # Rescale -1 to 1 X_train = (X_train.astype(np.float32) - 127.5) / 127.5 X_train = np.expand_dims(X_train, axis=3) y_train = y_train.reshape(-1, 1) # Adversarial ground truths valid = np.ones((batch_size, 1)) fake = np.zeros((batch_size, 1)) for epoch in range(epochs): # --------------------- # Train Discriminator # --------------------- # Select a random half batch of images idx = np.random.randint(0, X_train.shape[0], batch_size) imgs = X_train[idx] # Sample noise and categorical labels sampled_noise, sampled_labels = self.sample_generator_input(batch_size) gen_input = np.concatenate((sampled_noise, sampled_labels), axis=1) # Generate a half batch of new images gen_imgs = self.generator.predict(gen_input) # Train on real and generated data d_loss_real = self.discriminator.train_on_batch(imgs, valid) d_loss_fake = self.discriminator.train_on_batch(gen_imgs, fake) # Avg. loss d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # --------------------- # Train Generator and Q-network # --------------------- g_loss = self.combined.train_on_batch(gen_input, [valid, sampled_labels]) # Plot the progress print ("%d [D loss: %.2f, acc.: %.2f%%] [Q loss: %.2f] [G loss: %.2f]" % (epoch, d_loss[0], 100*d_loss[1], g_loss[1], g_loss[2])) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.sample_images(epoch) def sample_images(self, epoch): r, c = 10, 10 fig, axs = plt.subplots(r, c) for i in range(c): sampled_noise, _ = self.sample_generator_input(c) label = to_categorical(np.full(fill_value=i, shape=(r,1)), num_classes=self.num_classes) gen_input = np.concatenate((sampled_noise, label), axis=1) gen_imgs = self.generator.predict(gen_input) gen_imgs = 0.5 * gen_imgs + 0.5 for j in range(r): axs[j,i].imshow(gen_imgs[j,:,:,0], cmap='gray') axs[j,i].axis('off') fig.savefig("images/%d.png" % epoch) plt.close() def save_model(self): def save(model, model_name): model_path = "saved_model/%s.json" % model_name weights_path = "saved_model/%s_weights.hdf5" % model_name options = {"file_arch": model_path, "file_weight": weights_path} json_string = model.to_json() open(options['file_arch'], 'w').write(json_string) model.save_weights(options['file_weight']) save(self.generator, "generator") save(self.discriminator, "discriminator") if __name__ == '__main__': infogan = INFOGAN() infogan.train(epochs=50000, batch_size=128, sample_interval=50) ================================================ FILE: infogan/saved_model/.gitignore ================================================ * !.gitignore ================================================ FILE: lsgan/images/.gitignore ================================================ * !.gitignore ================================================ FILE: lsgan/lsgan.py ================================================ from __future__ import print_function, division from keras.datasets import mnist from keras.layers import Input, Dense, Reshape, Flatten, Dropout from keras.layers import BatchNormalization, Activation, ZeroPadding2D from keras.layers.advanced_activations import LeakyReLU from keras.layers.convolutional import UpSampling2D, Conv2D from keras.models import Sequential, Model from keras.optimizers import Adam import matplotlib.pyplot as plt import sys import numpy as np class LSGAN(): def __init__(self): self.img_rows = 28 self.img_cols = 28 self.channels = 1 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.latent_dim = 100 optimizer = Adam(0.0002, 0.5) # Build and compile the discriminator self.discriminator = self.build_discriminator() self.discriminator.compile(loss='mse', optimizer=optimizer, metrics=['accuracy']) # Build the generator self.generator = self.build_generator() # The generator takes noise as input and generated imgs z = Input(shape=(self.latent_dim,)) img = self.generator(z) # For the combined model we will only train the generator self.discriminator.trainable = False # The valid takes generated images as input and determines validity valid = self.discriminator(img) # The combined model (stacked generator and discriminator) # Trains generator to fool discriminator self.combined = Model(z, valid) # (!!!) Optimize w.r.t. MSE loss instead of crossentropy self.combined.compile(loss='mse', optimizer=optimizer) def build_generator(self): model = Sequential() model.add(Dense(256, input_dim=self.latent_dim)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(1024)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(np.prod(self.img_shape), activation='tanh')) model.add(Reshape(self.img_shape)) model.summary() noise = Input(shape=(self.latent_dim,)) img = model(noise) return Model(noise, img) def build_discriminator(self): model = Sequential() model.add(Flatten(input_shape=self.img_shape)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(256)) model.add(LeakyReLU(alpha=0.2)) # (!!!) No softmax model.add(Dense(1)) model.summary() img = Input(shape=self.img_shape) validity = model(img) return Model(img, validity) def train(self, epochs, batch_size=128, sample_interval=50): # Load the dataset (X_train, _), (_, _) = mnist.load_data() # Rescale -1 to 1 X_train = (X_train.astype(np.float32) - 127.5) / 127.5 X_train = np.expand_dims(X_train, axis=3) # Adversarial ground truths valid = np.ones((batch_size, 1)) fake = np.zeros((batch_size, 1)) for epoch in range(epochs): # --------------------- # Train Discriminator # --------------------- # Select a random batch of images idx = np.random.randint(0, X_train.shape[0], batch_size) imgs = X_train[idx] # Sample noise as generator input noise = np.random.normal(0, 1, (batch_size, self.latent_dim)) # Generate a batch of new images gen_imgs = self.generator.predict(noise) # Train the discriminator d_loss_real = self.discriminator.train_on_batch(imgs, valid) d_loss_fake = self.discriminator.train_on_batch(gen_imgs, fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # --------------------- # Train Generator # --------------------- g_loss = self.combined.train_on_batch(noise, valid) # Plot the progress print ("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]" % (epoch, d_loss[0], 100*d_loss[1], g_loss)) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.sample_images(epoch) def sample_images(self, epoch): r, c = 5, 5 noise = np.random.normal(0, 1, (r * c, self.latent_dim)) gen_imgs = self.generator.predict(noise) # Rescale images 0 - 1 gen_imgs = 0.5 * gen_imgs + 0.5 fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): axs[i,j].imshow(gen_imgs[cnt, :,:,0], cmap='gray') axs[i,j].axis('off') cnt += 1 fig.savefig("images/mnist_%d.png" % epoch) plt.close() if __name__ == '__main__': gan = LSGAN() gan.train(epochs=30000, batch_size=32, sample_interval=200) ================================================ FILE: lsgan/saved_model/.gitignore ================================================ * !.gitignore ================================================ FILE: pix2pix/data_loader.py ================================================ import scipy from glob import glob import numpy as np import matplotlib.pyplot as plt class DataLoader(): def __init__(self, dataset_name, img_res=(128, 128)): self.dataset_name = dataset_name self.img_res = img_res def load_data(self, batch_size=1, is_testing=False): data_type = "train" if not is_testing else "test" path = glob('./datasets/%s/%s/*' % (self.dataset_name, data_type)) batch_images = np.random.choice(path, size=batch_size) imgs_A = [] imgs_B = [] for img_path in batch_images: img = self.imread(img_path) h, w, _ = img.shape _w = int(w/2) img_A, img_B = img[:, :_w, :], img[:, _w:, :] img_A = scipy.misc.imresize(img_A, self.img_res) img_B = scipy.misc.imresize(img_B, self.img_res) # If training => do random flip if not is_testing and np.random.random() < 0.5: img_A = np.fliplr(img_A) img_B = np.fliplr(img_B) imgs_A.append(img_A) imgs_B.append(img_B) imgs_A = np.array(imgs_A)/127.5 - 1. imgs_B = np.array(imgs_B)/127.5 - 1. return imgs_A, imgs_B def load_batch(self, batch_size=1, is_testing=False): data_type = "train" if not is_testing else "val" path = glob('./datasets/%s/%s/*' % (self.dataset_name, data_type)) self.n_batches = int(len(path) / batch_size) for i in range(self.n_batches-1): batch = path[i*batch_size:(i+1)*batch_size] imgs_A, imgs_B = [], [] for img in batch: img = self.imread(img) h, w, _ = img.shape half_w = int(w/2) img_A = img[:, :half_w, :] img_B = img[:, half_w:, :] img_A = scipy.misc.imresize(img_A, self.img_res) img_B = scipy.misc.imresize(img_B, self.img_res) if not is_testing and np.random.random() > 0.5: img_A = np.fliplr(img_A) img_B = np.fliplr(img_B) imgs_A.append(img_A) imgs_B.append(img_B) imgs_A = np.array(imgs_A)/127.5 - 1. imgs_B = np.array(imgs_B)/127.5 - 1. yield imgs_A, imgs_B def imread(self, path): return scipy.misc.imread(path, mode='RGB').astype(np.float) ================================================ FILE: pix2pix/download_dataset.sh ================================================ mkdir datasets FILE=$1 URL=https://people.eecs.berkeley.edu/~tinghuiz/projects/pix2pix/datasets/$FILE.tar.gz TAR_FILE=./datasets/$FILE.tar.gz TARGET_DIR=./datasets/$FILE/ wget -N $URL -O $TAR_FILE mkdir $TARGET_DIR tar -zxvf $TAR_FILE -C ./datasets/ rm $TAR_FILE ================================================ FILE: pix2pix/images/.gitignore ================================================ * !.gitignore ================================================ FILE: pix2pix/pix2pix.py ================================================ from __future__ import print_function, division import scipy from keras.datasets import mnist from keras_contrib.layers.normalization.instancenormalization import InstanceNormalization from keras.layers import Input, Dense, Reshape, Flatten, Dropout, Concatenate from keras.layers import BatchNormalization, Activation, ZeroPadding2D from keras.layers.advanced_activations import LeakyReLU from keras.layers.convolutional import UpSampling2D, Conv2D from keras.models import Sequential, Model from keras.optimizers import Adam import datetime import matplotlib.pyplot as plt import sys from data_loader import DataLoader import numpy as np import os class Pix2Pix(): def __init__(self): # Input shape self.img_rows = 256 self.img_cols = 256 self.channels = 3 self.img_shape = (self.img_rows, self.img_cols, self.channels) # Configure data loader self.dataset_name = 'facades' self.data_loader = DataLoader(dataset_name=self.dataset_name, img_res=(self.img_rows, self.img_cols)) # Calculate output shape of D (PatchGAN) patch = int(self.img_rows / 2**4) self.disc_patch = (patch, patch, 1) # Number of filters in the first layer of G and D self.gf = 64 self.df = 64 optimizer = Adam(0.0002, 0.5) # Build and compile the discriminator self.discriminator = self.build_discriminator() self.discriminator.compile(loss='mse', optimizer=optimizer, metrics=['accuracy']) #------------------------- # Construct Computational # Graph of Generator #------------------------- # Build the generator self.generator = self.build_generator() # Input images and their conditioning images img_A = Input(shape=self.img_shape) img_B = Input(shape=self.img_shape) # By conditioning on B generate a fake version of A fake_A = self.generator(img_B) # For the combined model we will only train the generator self.discriminator.trainable = False # Discriminators determines validity of translated images / condition pairs valid = self.discriminator([fake_A, img_B]) self.combined = Model(inputs=[img_A, img_B], outputs=[valid, fake_A]) self.combined.compile(loss=['mse', 'mae'], loss_weights=[1, 100], optimizer=optimizer) def build_generator(self): """U-Net Generator""" def conv2d(layer_input, filters, f_size=4, bn=True): """Layers used during downsampling""" d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input) d = LeakyReLU(alpha=0.2)(d) if bn: d = BatchNormalization(momentum=0.8)(d) return d def deconv2d(layer_input, skip_input, filters, f_size=4, dropout_rate=0): """Layers used during upsampling""" u = UpSampling2D(size=2)(layer_input) u = Conv2D(filters, kernel_size=f_size, strides=1, padding='same', activation='relu')(u) if dropout_rate: u = Dropout(dropout_rate)(u) u = BatchNormalization(momentum=0.8)(u) u = Concatenate()([u, skip_input]) return u # Image input d0 = Input(shape=self.img_shape) # Downsampling d1 = conv2d(d0, self.gf, bn=False) d2 = conv2d(d1, self.gf*2) d3 = conv2d(d2, self.gf*4) d4 = conv2d(d3, self.gf*8) d5 = conv2d(d4, self.gf*8) d6 = conv2d(d5, self.gf*8) d7 = conv2d(d6, self.gf*8) # Upsampling u1 = deconv2d(d7, d6, self.gf*8) u2 = deconv2d(u1, d5, self.gf*8) u3 = deconv2d(u2, d4, self.gf*8) u4 = deconv2d(u3, d3, self.gf*4) u5 = deconv2d(u4, d2, self.gf*2) u6 = deconv2d(u5, d1, self.gf) u7 = UpSampling2D(size=2)(u6) output_img = Conv2D(self.channels, kernel_size=4, strides=1, padding='same', activation='tanh')(u7) return Model(d0, output_img) def build_discriminator(self): def d_layer(layer_input, filters, f_size=4, bn=True): """Discriminator layer""" d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input) d = LeakyReLU(alpha=0.2)(d) if bn: d = BatchNormalization(momentum=0.8)(d) return d img_A = Input(shape=self.img_shape) img_B = Input(shape=self.img_shape) # Concatenate image and conditioning image by channels to produce input combined_imgs = Concatenate(axis=-1)([img_A, img_B]) d1 = d_layer(combined_imgs, self.df, bn=False) d2 = d_layer(d1, self.df*2) d3 = d_layer(d2, self.df*4) d4 = d_layer(d3, self.df*8) validity = Conv2D(1, kernel_size=4, strides=1, padding='same')(d4) return Model([img_A, img_B], validity) def train(self, epochs, batch_size=1, sample_interval=50): start_time = datetime.datetime.now() # Adversarial loss ground truths valid = np.ones((batch_size,) + self.disc_patch) fake = np.zeros((batch_size,) + self.disc_patch) for epoch in range(epochs): for batch_i, (imgs_A, imgs_B) in enumerate(self.data_loader.load_batch(batch_size)): # --------------------- # Train Discriminator # --------------------- # Condition on B and generate a translated version fake_A = self.generator.predict(imgs_B) # Train the discriminators (original images = real / generated = Fake) d_loss_real = self.discriminator.train_on_batch([imgs_A, imgs_B], valid) d_loss_fake = self.discriminator.train_on_batch([fake_A, imgs_B], fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # ----------------- # Train Generator # ----------------- # Train the generators g_loss = self.combined.train_on_batch([imgs_A, imgs_B], [valid, imgs_A]) elapsed_time = datetime.datetime.now() - start_time # Plot the progress print ("[Epoch %d/%d] [Batch %d/%d] [D loss: %f, acc: %3d%%] [G loss: %f] time: %s" % (epoch, epochs, batch_i, self.data_loader.n_batches, d_loss[0], 100*d_loss[1], g_loss[0], elapsed_time)) # If at save interval => save generated image samples if batch_i % sample_interval == 0: self.sample_images(epoch, batch_i) def sample_images(self, epoch, batch_i): os.makedirs('images/%s' % self.dataset_name, exist_ok=True) r, c = 3, 3 imgs_A, imgs_B = self.data_loader.load_data(batch_size=3, is_testing=True) fake_A = self.generator.predict(imgs_B) gen_imgs = np.concatenate([imgs_B, fake_A, imgs_A]) # Rescale images 0 - 1 gen_imgs = 0.5 * gen_imgs + 0.5 titles = ['Condition', 'Generated', 'Original'] fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): axs[i,j].imshow(gen_imgs[cnt]) axs[i, j].set_title(titles[i]) axs[i,j].axis('off') cnt += 1 fig.savefig("images/%s/%d_%d.png" % (self.dataset_name, epoch, batch_i)) plt.close() if __name__ == '__main__': gan = Pix2Pix() gan.train(epochs=200, batch_size=1, sample_interval=200) ================================================ FILE: pix2pix/saved_model/.gitignore ================================================ * !.gitignore ================================================ FILE: pixelda/data_loader.py ================================================ import scipy from glob import glob import numpy as np from keras.datasets import mnist from skimage.transform import resize as imresize import pickle import os import urllib import gzip class DataLoader(): """Loads images from MNIST (domain A) and MNIST-M (domain B)""" def __init__(self, img_res=(128, 128)): self.img_res = img_res self.mnistm_url = 'https://github.com/VanushVaswani/keras_mnistm/releases/download/1.0/keras_mnistm.pkl.gz' self.setup_mnist(img_res) self.setup_mnistm(img_res) def normalize(self, images): return images.astype(np.float32) / 127.5 - 1. def setup_mnist(self, img_res): print ("Setting up MNIST...") if not os.path.exists('datasets/mnist_x.npy'): # Load the dataset (mnist_X, mnist_y), (_, _) = mnist.load_data() # Normalize and rescale images mnist_X = self.normalize(mnist_X) mnist_X = np.array([imresize(x, img_res) for x in mnist_X]) mnist_X = np.expand_dims(mnist_X, axis=-1) mnist_X = np.repeat(mnist_X, 3, axis=-1) self.mnist_X, self.mnist_y = mnist_X, mnist_y # Save formatted images np.save('datasets/mnist_x.npy', self.mnist_X) np.save('datasets/mnist_y.npy', self.mnist_y) else: self.mnist_X = np.load('datasets/mnist_x.npy') self.mnist_y = np.load('datasets/mnist_y.npy') print ("+ Done.") def setup_mnistm(self, img_res): print ("Setting up MNIST-M...") if not os.path.exists('datasets/mnistm_x.npy'): # Download the MNIST-M pkl file filepath = 'datasets/keras_mnistm.pkl.gz' if not os.path.exists(filepath.replace('.gz', '')): print('+ Downloading ' + self.mnistm_url) data = urllib.request.urlopen(self.mnistm_url) with open(filepath, 'wb') as f: f.write(data.read()) with open(filepath.replace('.gz', ''), 'wb') as out_f, \ gzip.GzipFile(filepath) as zip_f: out_f.write(zip_f.read()) os.unlink(filepath) # load MNIST-M images from pkl file with open('datasets/keras_mnistm.pkl', "rb") as f: data = pickle.load(f, encoding='bytes') # Normalize and rescale images mnistm_X = np.array(data[b'train']) mnistm_X = self.normalize(mnistm_X) mnistm_X = np.array([imresize(x, img_res) for x in mnistm_X]) self.mnistm_X, self.mnistm_y = mnistm_X, self.mnist_y.copy() # Save formatted images np.save('datasets/mnistm_x.npy', self.mnistm_X) np.save('datasets/mnistm_y.npy', self.mnistm_y) else: self.mnistm_X = np.load('datasets/mnistm_x.npy') self.mnistm_y = np.load('datasets/mnistm_y.npy') print ("+ Done.") def load_data(self, domain, batch_size=1): X = self.mnist_X if domain == 'A' else self.mnistm_X y = self.mnist_y if domain == 'A' else self.mnistm_y idx = np.random.choice(list(range(len(X))), size=batch_size) return X[idx], y[idx] ================================================ FILE: pixelda/images/.gitignore ================================================ * !.gitignore ================================================ FILE: pixelda/pixelda.py ================================================ from __future__ import print_function, division import scipy from keras.datasets import mnist from keras_contrib.layers.normalization.instancenormalization import InstanceNormalization from keras.layers import Input, Dense, Reshape, Flatten, Dropout, Concatenate from keras.layers import BatchNormalization, Activation, ZeroPadding2D, Add from keras.layers.advanced_activations import LeakyReLU from keras.layers.convolutional import UpSampling2D, Conv2D from keras.models import Sequential, Model from keras.optimizers import Adam from keras.utils import to_categorical import datetime import matplotlib.pyplot as plt import sys from data_loader import DataLoader import numpy as np import os class PixelDA(): def __init__(self): # Input shape self.img_rows = 32 self.img_cols = 32 self.channels = 3 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.num_classes = 10 # Configure MNIST and MNIST-M data loader self.data_loader = DataLoader(img_res=(self.img_rows, self.img_cols)) # Loss weights lambda_adv = 10 lambda_clf = 1 # Calculate output shape of D (PatchGAN) patch = int(self.img_rows / 2**4) self.disc_patch = (patch, patch, 1) # Number of residual blocks in the generator self.residual_blocks = 6 optimizer = Adam(0.0002, 0.5) # Number of filters in first layer of discriminator and classifier self.df = 64 self.cf = 64 # Build and compile the discriminators self.discriminator = self.build_discriminator() self.discriminator.compile(loss='mse', optimizer=optimizer, metrics=['accuracy']) # Build the generator self.generator = self.build_generator() # Build the task (classification) network self.clf = self.build_classifier() # Input images from both domains img_A = Input(shape=self.img_shape) img_B = Input(shape=self.img_shape) # Translate images from domain A to domain B fake_B = self.generator(img_A) # Classify the translated image class_pred = self.clf(fake_B) # For the combined model we will only train the generator and classifier self.discriminator.trainable = False # Discriminator determines validity of translated images valid = self.discriminator(fake_B) self.combined = Model(img_A, [valid, class_pred]) self.combined.compile(loss=['mse', 'categorical_crossentropy'], loss_weights=[lambda_adv, lambda_clf], optimizer=optimizer, metrics=['accuracy']) def build_generator(self): """Resnet Generator""" def residual_block(layer_input): """Residual block described in paper""" d = Conv2D(64, kernel_size=3, strides=1, padding='same')(layer_input) d = BatchNormalization(momentum=0.8)(d) d = Activation('relu')(d) d = Conv2D(64, kernel_size=3, strides=1, padding='same')(d) d = BatchNormalization(momentum=0.8)(d) d = Add()([d, layer_input]) return d # Image input img = Input(shape=self.img_shape) l1 = Conv2D(64, kernel_size=3, padding='same', activation='relu')(img) # Propogate signal through residual blocks r = residual_block(l1) for _ in range(self.residual_blocks - 1): r = residual_block(r) output_img = Conv2D(self.channels, kernel_size=3, padding='same', activation='tanh')(r) return Model(img, output_img) def build_discriminator(self): def d_layer(layer_input, filters, f_size=4, normalization=True): """Discriminator layer""" d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input) d = LeakyReLU(alpha=0.2)(d) if normalization: d = InstanceNormalization()(d) return d img = Input(shape=self.img_shape) d1 = d_layer(img, self.df, normalization=False) d2 = d_layer(d1, self.df*2) d3 = d_layer(d2, self.df*4) d4 = d_layer(d3, self.df*8) validity = Conv2D(1, kernel_size=4, strides=1, padding='same')(d4) return Model(img, validity) def build_classifier(self): def clf_layer(layer_input, filters, f_size=4, normalization=True): """Classifier layer""" d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input) d = LeakyReLU(alpha=0.2)(d) if normalization: d = InstanceNormalization()(d) return d img = Input(shape=self.img_shape) c1 = clf_layer(img, self.cf, normalization=False) c2 = clf_layer(c1, self.cf*2) c3 = clf_layer(c2, self.cf*4) c4 = clf_layer(c3, self.cf*8) c5 = clf_layer(c4, self.cf*8) class_pred = Dense(self.num_classes, activation='softmax')(Flatten()(c5)) return Model(img, class_pred) def train(self, epochs, batch_size=128, sample_interval=50): half_batch = int(batch_size / 2) # Classification accuracy on 100 last batches of domain B test_accs = [] # Adversarial ground truths valid = np.ones((batch_size, *self.disc_patch)) fake = np.zeros((batch_size, *self.disc_patch)) for epoch in range(epochs): # --------------------- # Train Discriminator # --------------------- imgs_A, labels_A = self.data_loader.load_data(domain="A", batch_size=batch_size) imgs_B, labels_B = self.data_loader.load_data(domain="B", batch_size=batch_size) # Translate images from domain A to domain B fake_B = self.generator.predict(imgs_A) # Train the discriminators (original images = real / translated = Fake) d_loss_real = self.discriminator.train_on_batch(imgs_B, valid) d_loss_fake = self.discriminator.train_on_batch(fake_B, fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # -------------------------------- # Train Generator and Classifier # -------------------------------- # One-hot encoding of labels labels_A = to_categorical(labels_A, num_classes=self.num_classes) # Train the generator and classifier g_loss = self.combined.train_on_batch(imgs_A, [valid, labels_A]) #----------------------- # Evaluation (domain B) #----------------------- pred_B = self.clf.predict(imgs_B) test_acc = np.mean(np.argmax(pred_B, axis=1) == labels_B) # Add accuracy to list of last 100 accuracy measurements test_accs.append(test_acc) if len(test_accs) > 100: test_accs.pop(0) # Plot the progress print ( "%d : [D - loss: %.5f, acc: %3d%%], [G - loss: %.5f], [clf - loss: %.5f, acc: %3d%%, test_acc: %3d%% (%3d%%)]" % \ (epoch, d_loss[0], 100*float(d_loss[1]), g_loss[1], g_loss[2], 100*float(g_loss[-1]), 100*float(test_acc), 100*float(np.mean(test_accs)))) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.sample_images(epoch) def sample_images(self, epoch): r, c = 2, 5 imgs_A, _ = self.data_loader.load_data(domain="A", batch_size=5) # Translate images to the other domain fake_B = self.generator.predict(imgs_A) gen_imgs = np.concatenate([imgs_A, fake_B]) # Rescale images 0 - 1 gen_imgs = 0.5 * gen_imgs + 0.5 #titles = ['Original', 'Translated'] fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): axs[i,j].imshow(gen_imgs[cnt]) #axs[i, j].set_title(titles[i]) axs[i,j].axis('off') cnt += 1 fig.savefig("images/%d.png" % (epoch)) plt.close() if __name__ == '__main__': gan = PixelDA() gan.train(epochs=30000, batch_size=32, sample_interval=500) ================================================ FILE: pixelda/saved_model/.gitignore ================================================ * !.gitignore ================================================ FILE: pixelda/test.py ================================================ from __future__ import print_function, division import scipy import datetime import matplotlib.pyplot as plt import sys from data_loader import DataLoader import numpy as np import os # Configure MNIST and MNIST-M data loader data_loader = DataLoader(img_res=(32, 32)) mnist, _ = data_loader.load_data(domain="A", batch_size=25) mnistm, _ = data_loader.load_data(domain="B", batch_size=25) r, c = 5, 5 for img_i, imgs in enumerate([mnist, mnistm]): #titles = ['Original', 'Translated'] fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): axs[i,j].imshow(imgs[cnt]) #axs[i, j].set_title(titles[i]) axs[i,j].axis('off') cnt += 1 fig.savefig("%d.png" % (img_i)) plt.close() ================================================ FILE: requirements.txt ================================================ keras git+https://www.github.com/keras-team/keras-contrib.git matplotlib numpy scipy pillow #urllib #skimage scikit-image #gzip #pickle ================================================ FILE: sgan/images/.gitignore ================================================ * !.gitignore ================================================ FILE: sgan/saved_model/.gitignore ================================================ * !.gitignore ================================================ FILE: sgan/sgan.py ================================================ from __future__ import print_function, division from keras.datasets import mnist from keras.layers import Input, Dense, Reshape, Flatten, Dropout, multiply, GaussianNoise from keras.layers import BatchNormalization, Activation, Embedding, ZeroPadding2D from keras.layers.advanced_activations import LeakyReLU from keras.layers.convolutional import UpSampling2D, Conv2D from keras.models import Sequential, Model from keras.optimizers import Adam from keras import losses from keras.utils import to_categorical import keras.backend as K import matplotlib.pyplot as plt import numpy as np class SGAN: def __init__(self): self.img_rows = 28 self.img_cols = 28 self.channels = 1 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.num_classes = 10 self.latent_dim = 100 optimizer = Adam(0.0002, 0.5) # Build and compile the discriminator self.discriminator = self.build_discriminator() self.discriminator.compile( loss=['binary_crossentropy', 'categorical_crossentropy'], loss_weights=[0.5, 0.5], optimizer=optimizer, metrics=['accuracy'] ) # Build the generator self.generator = self.build_generator() # The generator takes noise as input and generates imgs noise = Input(shape=(100,)) img = self.generator(noise) # For the combined model we will only train the generator self.discriminator.trainable = False # The valid takes generated images as input and determines validity valid, _ = self.discriminator(img) # The combined model (stacked generator and discriminator) # Trains generator to fool discriminator self.combined = Model(noise, valid) self.combined.compile(loss=['binary_crossentropy'], optimizer=optimizer) def build_generator(self): model = Sequential() model.add(Dense(128 * 7 * 7, activation="relu", input_dim=self.latent_dim)) model.add(Reshape((7, 7, 128))) model.add(BatchNormalization(momentum=0.8)) model.add(UpSampling2D()) model.add(Conv2D(128, kernel_size=3, padding="same")) model.add(Activation("relu")) model.add(BatchNormalization(momentum=0.8)) model.add(UpSampling2D()) model.add(Conv2D(64, kernel_size=3, padding="same")) model.add(Activation("relu")) model.add(BatchNormalization(momentum=0.8)) model.add(Conv2D(1, kernel_size=3, padding="same")) model.add(Activation("tanh")) model.summary() noise = Input(shape=(self.latent_dim,)) img = model(noise) return Model(noise, img) def build_discriminator(self): model = Sequential() model.add(Conv2D(32, kernel_size=3, strides=2, input_shape=self.img_shape, padding="same")) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Conv2D(64, kernel_size=3, strides=2, padding="same")) model.add(ZeroPadding2D(padding=((0,1),(0,1)))) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(BatchNormalization(momentum=0.8)) model.add(Conv2D(128, kernel_size=3, strides=2, padding="same")) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(BatchNormalization(momentum=0.8)) model.add(Conv2D(256, kernel_size=3, strides=1, padding="same")) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Flatten()) model.summary() img = Input(shape=self.img_shape) features = model(img) valid = Dense(1, activation="sigmoid")(features) label = Dense(self.num_classes+1, activation="softmax")(features) return Model(img, [valid, label]) def train(self, epochs, batch_size=128, sample_interval=50): # Load the dataset (X_train, y_train), (_, _) = mnist.load_data() # Rescale -1 to 1 X_train = (X_train.astype(np.float32) - 127.5) / 127.5 X_train = np.expand_dims(X_train, axis=3) y_train = y_train.reshape(-1, 1) # Class weights: # To balance the difference in occurences of digit class labels. # 50% of labels that the discriminator trains on are 'fake'. # Weight = 1 / frequency half_batch = batch_size // 2 cw1 = {0: 1, 1: 1} cw2 = {i: self.num_classes / half_batch for i in range(self.num_classes)} cw2[self.num_classes] = 1 / half_batch # Adversarial ground truths valid = np.ones((batch_size, 1)) fake = np.zeros((batch_size, 1)) for epoch in range(epochs): # --------------------- # Train Discriminator # --------------------- # Select a random batch of images idx = np.random.randint(0, X_train.shape[0], batch_size) imgs = X_train[idx] # Sample noise and generate a batch of new images noise = np.random.normal(0, 1, (batch_size, self.latent_dim)) gen_imgs = self.generator.predict(noise) # One-hot encoding of labels labels = to_categorical(y_train[idx], num_classes=self.num_classes+1) fake_labels = to_categorical(np.full((batch_size, 1), self.num_classes), num_classes=self.num_classes+1) # Train the discriminator d_loss_real = self.discriminator.train_on_batch(imgs, [valid, labels], class_weight=[cw1, cw2]) d_loss_fake = self.discriminator.train_on_batch(gen_imgs, [fake, fake_labels], class_weight=[cw1, cw2]) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # --------------------- # Train Generator # --------------------- g_loss = self.combined.train_on_batch(noise, valid, class_weight=[cw1, cw2]) # Plot the progress print ("%d [D loss: %f, acc: %.2f%%, op_acc: %.2f%%] [G loss: %f]" % (epoch, d_loss[0], 100*d_loss[3], 100*d_loss[4], g_loss)) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.sample_images(epoch) def sample_images(self, epoch): r, c = 5, 5 noise = np.random.normal(0, 1, (r * c, self.latent_dim)) gen_imgs = self.generator.predict(noise) # Rescale images 0 - 1 gen_imgs = 0.5 * gen_imgs + 0.5 fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): axs[i,j].imshow(gen_imgs[cnt, :,:,0], cmap='gray') axs[i,j].axis('off') cnt += 1 fig.savefig("images/mnist_%d.png" % epoch) plt.close() def save_model(self): def save(model, model_name): model_path = "saved_model/%s.json" % model_name weights_path = "saved_model/%s_weights.hdf5" % model_name options = {"file_arch": model_path, "file_weight": weights_path} json_string = model.to_json() open(options['file_arch'], 'w').write(json_string) model.save_weights(options['file_weight']) save(self.generator, "mnist_sgan_generator") save(self.discriminator, "mnist_sgan_discriminator") save(self.combined, "mnist_sgan_adversarial") if __name__ == '__main__': sgan = SGAN() sgan.train(epochs=20000, batch_size=32, sample_interval=50) ================================================ FILE: srgan/data_loader.py ================================================ import scipy from glob import glob import numpy as np import matplotlib.pyplot as plt class DataLoader(): def __init__(self, dataset_name, img_res=(128, 128)): self.dataset_name = dataset_name self.img_res = img_res def load_data(self, batch_size=1, is_testing=False): data_type = "train" if not is_testing else "test" path = glob('./datasets/%s/*' % (self.dataset_name)) batch_images = np.random.choice(path, size=batch_size) imgs_hr = [] imgs_lr = [] for img_path in batch_images: img = self.imread(img_path) h, w = self.img_res low_h, low_w = int(h / 4), int(w / 4) img_hr = scipy.misc.imresize(img, self.img_res) img_lr = scipy.misc.imresize(img, (low_h, low_w)) # If training => do random flip if not is_testing and np.random.random() < 0.5: img_hr = np.fliplr(img_hr) img_lr = np.fliplr(img_lr) imgs_hr.append(img_hr) imgs_lr.append(img_lr) imgs_hr = np.array(imgs_hr) / 127.5 - 1. imgs_lr = np.array(imgs_lr) / 127.5 - 1. return imgs_hr, imgs_lr def imread(self, path): return scipy.misc.imread(path, mode='RGB').astype(np.float) ================================================ FILE: srgan/images/.gitignore ================================================ * !.gitignore ================================================ FILE: srgan/saved_model/.gitignore ================================================ * !.gitignore ================================================ FILE: srgan/srgan.py ================================================ """ Super-resolution of CelebA using Generative Adversarial Networks. The dataset can be downloaded from: https://www.dropbox.com/sh/8oqt9vytwxb3s4r/AADIKlz8PR9zr6Y20qbkunrba/Img/img_align_celeba.zip?dl=0 Instrustion on running the script: 1. Download the dataset from the provided link 2. Save the folder 'img_align_celeba' to 'datasets/' 4. Run the sript using command 'python srgan.py' """ from __future__ import print_function, division import scipy from keras.datasets import mnist from keras_contrib.layers.normalization.instancenormalization import InstanceNormalization from keras.layers import Input, Dense, Reshape, Flatten, Dropout, Concatenate from keras.layers import BatchNormalization, Activation, ZeroPadding2D, Add from keras.layers.advanced_activations import PReLU, LeakyReLU from keras.layers.convolutional import UpSampling2D, Conv2D from keras.applications import VGG19 from keras.models import Sequential, Model from keras.optimizers import Adam import datetime import matplotlib.pyplot as plt import sys from data_loader import DataLoader import numpy as np import os import keras.backend as K class SRGAN(): def __init__(self): # Input shape self.channels = 3 self.lr_height = 64 # Low resolution height self.lr_width = 64 # Low resolution width self.lr_shape = (self.lr_height, self.lr_width, self.channels) self.hr_height = self.lr_height*4 # High resolution height self.hr_width = self.lr_width*4 # High resolution width self.hr_shape = (self.hr_height, self.hr_width, self.channels) # Number of residual blocks in the generator self.n_residual_blocks = 16 optimizer = Adam(0.0002, 0.5) # We use a pre-trained VGG19 model to extract image features from the high resolution # and the generated high resolution images and minimize the mse between them self.vgg = self.build_vgg() self.vgg.trainable = False self.vgg.compile(loss='mse', optimizer=optimizer, metrics=['accuracy']) # Configure data loader self.dataset_name = 'img_align_celeba' self.data_loader = DataLoader(dataset_name=self.dataset_name, img_res=(self.hr_height, self.hr_width)) # Calculate output shape of D (PatchGAN) patch = int(self.hr_height / 2**4) self.disc_patch = (patch, patch, 1) # Number of filters in the first layer of G and D self.gf = 64 self.df = 64 # Build and compile the discriminator self.discriminator = self.build_discriminator() self.discriminator.compile(loss='mse', optimizer=optimizer, metrics=['accuracy']) # Build the generator self.generator = self.build_generator() # High res. and low res. images img_hr = Input(shape=self.hr_shape) img_lr = Input(shape=self.lr_shape) # Generate high res. version from low res. fake_hr = self.generator(img_lr) # Extract image features of the generated img fake_features = self.vgg(fake_hr) # For the combined model we will only train the generator self.discriminator.trainable = False # Discriminator determines validity of generated high res. images validity = self.discriminator(fake_hr) self.combined = Model([img_lr, img_hr], [validity, fake_features]) self.combined.compile(loss=['binary_crossentropy', 'mse'], loss_weights=[1e-3, 1], optimizer=optimizer) def build_vgg(self): """ Builds a pre-trained VGG19 model that outputs image features extracted at the third block of the model """ vgg = VGG19(weights="imagenet") # Set outputs to outputs of last conv. layer in block 3 # See architecture at: https://github.com/keras-team/keras/blob/master/keras/applications/vgg19.py vgg.outputs = [vgg.layers[9].output] img = Input(shape=self.hr_shape) # Extract image features img_features = vgg(img) return Model(img, img_features) def build_generator(self): def residual_block(layer_input, filters): """Residual block described in paper""" d = Conv2D(filters, kernel_size=3, strides=1, padding='same')(layer_input) d = Activation('relu')(d) d = BatchNormalization(momentum=0.8)(d) d = Conv2D(filters, kernel_size=3, strides=1, padding='same')(d) d = BatchNormalization(momentum=0.8)(d) d = Add()([d, layer_input]) return d def deconv2d(layer_input): """Layers used during upsampling""" u = UpSampling2D(size=2)(layer_input) u = Conv2D(256, kernel_size=3, strides=1, padding='same')(u) u = Activation('relu')(u) return u # Low resolution image input img_lr = Input(shape=self.lr_shape) # Pre-residual block c1 = Conv2D(64, kernel_size=9, strides=1, padding='same')(img_lr) c1 = Activation('relu')(c1) # Propogate through residual blocks r = residual_block(c1, self.gf) for _ in range(self.n_residual_blocks - 1): r = residual_block(r, self.gf) # Post-residual block c2 = Conv2D(64, kernel_size=3, strides=1, padding='same')(r) c2 = BatchNormalization(momentum=0.8)(c2) c2 = Add()([c2, c1]) # Upsampling u1 = deconv2d(c2) u2 = deconv2d(u1) # Generate high resolution output gen_hr = Conv2D(self.channels, kernel_size=9, strides=1, padding='same', activation='tanh')(u2) return Model(img_lr, gen_hr) def build_discriminator(self): def d_block(layer_input, filters, strides=1, bn=True): """Discriminator layer""" d = Conv2D(filters, kernel_size=3, strides=strides, padding='same')(layer_input) d = LeakyReLU(alpha=0.2)(d) if bn: d = BatchNormalization(momentum=0.8)(d) return d # Input img d0 = Input(shape=self.hr_shape) d1 = d_block(d0, self.df, bn=False) d2 = d_block(d1, self.df, strides=2) d3 = d_block(d2, self.df*2) d4 = d_block(d3, self.df*2, strides=2) d5 = d_block(d4, self.df*4) d6 = d_block(d5, self.df*4, strides=2) d7 = d_block(d6, self.df*8) d8 = d_block(d7, self.df*8, strides=2) d9 = Dense(self.df*16)(d8) d10 = LeakyReLU(alpha=0.2)(d9) validity = Dense(1, activation='sigmoid')(d10) return Model(d0, validity) def train(self, epochs, batch_size=1, sample_interval=50): start_time = datetime.datetime.now() for epoch in range(epochs): # ---------------------- # Train Discriminator # ---------------------- # Sample images and their conditioning counterparts imgs_hr, imgs_lr = self.data_loader.load_data(batch_size) # From low res. image generate high res. version fake_hr = self.generator.predict(imgs_lr) valid = np.ones((batch_size,) + self.disc_patch) fake = np.zeros((batch_size,) + self.disc_patch) # Train the discriminators (original images = real / generated = Fake) d_loss_real = self.discriminator.train_on_batch(imgs_hr, valid) d_loss_fake = self.discriminator.train_on_batch(fake_hr, fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # ------------------ # Train Generator # ------------------ # Sample images and their conditioning counterparts imgs_hr, imgs_lr = self.data_loader.load_data(batch_size) # The generators want the discriminators to label the generated images as real valid = np.ones((batch_size,) + self.disc_patch) # Extract ground truth image features using pre-trained VGG19 model image_features = self.vgg.predict(imgs_hr) # Train the generators g_loss = self.combined.train_on_batch([imgs_lr, imgs_hr], [valid, image_features]) elapsed_time = datetime.datetime.now() - start_time # Plot the progress print ("%d time: %s" % (epoch, elapsed_time)) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.sample_images(epoch) def sample_images(self, epoch): os.makedirs('images/%s' % self.dataset_name, exist_ok=True) r, c = 2, 2 imgs_hr, imgs_lr = self.data_loader.load_data(batch_size=2, is_testing=True) fake_hr = self.generator.predict(imgs_lr) # Rescale images 0 - 1 imgs_lr = 0.5 * imgs_lr + 0.5 fake_hr = 0.5 * fake_hr + 0.5 imgs_hr = 0.5 * imgs_hr + 0.5 # Save generated images and the high resolution originals titles = ['Generated', 'Original'] fig, axs = plt.subplots(r, c) cnt = 0 for row in range(r): for col, image in enumerate([fake_hr, imgs_hr]): axs[row, col].imshow(image[row]) axs[row, col].set_title(titles[col]) axs[row, col].axis('off') cnt += 1 fig.savefig("images/%s/%d.png" % (self.dataset_name, epoch)) plt.close() # Save low resolution images for comparison for i in range(r): fig = plt.figure() plt.imshow(imgs_lr[i]) fig.savefig('images/%s/%d_lowres%d.png' % (self.dataset_name, epoch, i)) plt.close() if __name__ == '__main__': gan = SRGAN() gan.train(epochs=30000, batch_size=1, sample_interval=50) ================================================ FILE: wgan/images/.gitignore ================================================ * !.gitignore ================================================ FILE: wgan/saved_model/.gitignore ================================================ * !.gitignore ================================================ FILE: wgan/wgan.py ================================================ from __future__ import print_function, division from keras.datasets import mnist from keras.layers import Input, Dense, Reshape, Flatten, Dropout from keras.layers import BatchNormalization, Activation, ZeroPadding2D from keras.layers.advanced_activations import LeakyReLU from keras.layers.convolutional import UpSampling2D, Conv2D from keras.models import Sequential, Model from keras.optimizers import RMSprop import keras.backend as K import matplotlib.pyplot as plt import sys import numpy as np class WGAN(): def __init__(self): self.img_rows = 28 self.img_cols = 28 self.channels = 1 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.latent_dim = 100 # Following parameter and optimizer set as recommended in paper self.n_critic = 5 self.clip_value = 0.01 optimizer = RMSprop(lr=0.00005) # Build and compile the critic self.critic = self.build_critic() self.critic.compile(loss=self.wasserstein_loss, optimizer=optimizer, metrics=['accuracy']) # Build the generator self.generator = self.build_generator() # The generator takes noise as input and generated imgs z = Input(shape=(self.latent_dim,)) img = self.generator(z) # For the combined model we will only train the generator self.critic.trainable = False # The critic takes generated images as input and determines validity valid = self.critic(img) # The combined model (stacked generator and critic) self.combined = Model(z, valid) self.combined.compile(loss=self.wasserstein_loss, optimizer=optimizer, metrics=['accuracy']) def wasserstein_loss(self, y_true, y_pred): return K.mean(y_true * y_pred) def build_generator(self): model = Sequential() model.add(Dense(128 * 7 * 7, activation="relu", input_dim=self.latent_dim)) model.add(Reshape((7, 7, 128))) model.add(UpSampling2D()) model.add(Conv2D(128, kernel_size=4, padding="same")) model.add(BatchNormalization(momentum=0.8)) model.add(Activation("relu")) model.add(UpSampling2D()) model.add(Conv2D(64, kernel_size=4, padding="same")) model.add(BatchNormalization(momentum=0.8)) model.add(Activation("relu")) model.add(Conv2D(self.channels, kernel_size=4, padding="same")) model.add(Activation("tanh")) model.summary() noise = Input(shape=(self.latent_dim,)) img = model(noise) return Model(noise, img) def build_critic(self): model = Sequential() model.add(Conv2D(16, kernel_size=3, strides=2, input_shape=self.img_shape, padding="same")) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Conv2D(32, kernel_size=3, strides=2, padding="same")) model.add(ZeroPadding2D(padding=((0,1),(0,1)))) model.add(BatchNormalization(momentum=0.8)) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Conv2D(64, kernel_size=3, strides=2, padding="same")) model.add(BatchNormalization(momentum=0.8)) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Conv2D(128, kernel_size=3, strides=1, padding="same")) model.add(BatchNormalization(momentum=0.8)) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(1)) model.summary() img = Input(shape=self.img_shape) validity = model(img) return Model(img, validity) def train(self, epochs, batch_size=128, sample_interval=50): # Load the dataset (X_train, _), (_, _) = mnist.load_data() # Rescale -1 to 1 X_train = (X_train.astype(np.float32) - 127.5) / 127.5 X_train = np.expand_dims(X_train, axis=3) # Adversarial ground truths valid = -np.ones((batch_size, 1)) fake = np.ones((batch_size, 1)) for epoch in range(epochs): for _ in range(self.n_critic): # --------------------- # Train Discriminator # --------------------- # Select a random batch of images idx = np.random.randint(0, X_train.shape[0], batch_size) imgs = X_train[idx] # Sample noise as generator input noise = np.random.normal(0, 1, (batch_size, self.latent_dim)) # Generate a batch of new images gen_imgs = self.generator.predict(noise) # Train the critic d_loss_real = self.critic.train_on_batch(imgs, valid) d_loss_fake = self.critic.train_on_batch(gen_imgs, fake) d_loss = 0.5 * np.add(d_loss_fake, d_loss_real) # Clip critic weights for l in self.critic.layers: weights = l.get_weights() weights = [np.clip(w, -self.clip_value, self.clip_value) for w in weights] l.set_weights(weights) # --------------------- # Train Generator # --------------------- g_loss = self.combined.train_on_batch(noise, valid) # Plot the progress print ("%d [D loss: %f] [G loss: %f]" % (epoch, 1 - d_loss[0], 1 - g_loss[0])) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.sample_images(epoch) def sample_images(self, epoch): r, c = 5, 5 noise = np.random.normal(0, 1, (r * c, self.latent_dim)) gen_imgs = self.generator.predict(noise) # Rescale images 0 - 1 gen_imgs = 0.5 * gen_imgs + 0.5 fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): axs[i,j].imshow(gen_imgs[cnt, :,:,0], cmap='gray') axs[i,j].axis('off') cnt += 1 fig.savefig("images/mnist_%d.png" % epoch) plt.close() if __name__ == '__main__': wgan = WGAN() wgan.train(epochs=4000, batch_size=32, sample_interval=50) ================================================ FILE: wgan_gp/images/.gitignore ================================================ * !.gitignore ================================================ FILE: wgan_gp/saved_model/.gitignore ================================================ * !.gitignore ================================================ FILE: wgan_gp/wgan_gp.py ================================================ # Large amount of credit goes to: # https://github.com/keras-team/keras-contrib/blob/master/examples/improved_wgan.py # which I've used as a reference for this implementation from __future__ import print_function, division from keras.datasets import mnist from keras.layers.merge import _Merge from keras.layers import Input, Dense, Reshape, Flatten, Dropout from keras.layers import BatchNormalization, Activation, ZeroPadding2D from keras.layers.advanced_activations import LeakyReLU from keras.layers.convolutional import UpSampling2D, Conv2D from keras.models import Sequential, Model from keras.optimizers import RMSprop from functools import partial import keras.backend as K import matplotlib.pyplot as plt import sys import numpy as np class RandomWeightedAverage(_Merge): """Provides a (random) weighted average between real and generated image samples""" def _merge_function(self, inputs): alpha = K.random_uniform((32, 1, 1, 1)) return (alpha * inputs[0]) + ((1 - alpha) * inputs[1]) class WGANGP(): def __init__(self): self.img_rows = 28 self.img_cols = 28 self.channels = 1 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.latent_dim = 100 # Following parameter and optimizer set as recommended in paper self.n_critic = 5 optimizer = RMSprop(lr=0.00005) # Build the generator and critic self.generator = self.build_generator() self.critic = self.build_critic() #------------------------------- # Construct Computational Graph # for the Critic #------------------------------- # Freeze generator's layers while training critic self.generator.trainable = False # Image input (real sample) real_img = Input(shape=self.img_shape) # Noise input z_disc = Input(shape=(self.latent_dim,)) # Generate image based of noise (fake sample) fake_img = self.generator(z_disc) # Discriminator determines validity of the real and fake images fake = self.critic(fake_img) valid = self.critic(real_img) # Construct weighted average between real and fake images interpolated_img = RandomWeightedAverage()([real_img, fake_img]) # Determine validity of weighted sample validity_interpolated = self.critic(interpolated_img) # Use Python partial to provide loss function with additional # 'averaged_samples' argument partial_gp_loss = partial(self.gradient_penalty_loss, averaged_samples=interpolated_img) partial_gp_loss.__name__ = 'gradient_penalty' # Keras requires function names self.critic_model = Model(inputs=[real_img, z_disc], outputs=[valid, fake, validity_interpolated]) self.critic_model.compile(loss=[self.wasserstein_loss, self.wasserstein_loss, partial_gp_loss], optimizer=optimizer, loss_weights=[1, 1, 10]) #------------------------------- # Construct Computational Graph # for Generator #------------------------------- # For the generator we freeze the critic's layers self.critic.trainable = False self.generator.trainable = True # Sampled noise for input to generator z_gen = Input(shape=(self.latent_dim,)) # Generate images based of noise img = self.generator(z_gen) # Discriminator determines validity valid = self.critic(img) # Defines generator model self.generator_model = Model(z_gen, valid) self.generator_model.compile(loss=self.wasserstein_loss, optimizer=optimizer) def gradient_penalty_loss(self, y_true, y_pred, averaged_samples): """ Computes gradient penalty based on prediction and weighted real / fake samples """ gradients = K.gradients(y_pred, averaged_samples)[0] # compute the euclidean norm by squaring ... gradients_sqr = K.square(gradients) # ... summing over the rows ... gradients_sqr_sum = K.sum(gradients_sqr, axis=np.arange(1, len(gradients_sqr.shape))) # ... and sqrt gradient_l2_norm = K.sqrt(gradients_sqr_sum) # compute lambda * (1 - ||grad||)^2 still for each single sample gradient_penalty = K.square(1 - gradient_l2_norm) # return the mean as loss over all the batch samples return K.mean(gradient_penalty) def wasserstein_loss(self, y_true, y_pred): return K.mean(y_true * y_pred) def build_generator(self): model = Sequential() model.add(Dense(128 * 7 * 7, activation="relu", input_dim=self.latent_dim)) model.add(Reshape((7, 7, 128))) model.add(UpSampling2D()) model.add(Conv2D(128, kernel_size=4, padding="same")) model.add(BatchNormalization(momentum=0.8)) model.add(Activation("relu")) model.add(UpSampling2D()) model.add(Conv2D(64, kernel_size=4, padding="same")) model.add(BatchNormalization(momentum=0.8)) model.add(Activation("relu")) model.add(Conv2D(self.channels, kernel_size=4, padding="same")) model.add(Activation("tanh")) model.summary() noise = Input(shape=(self.latent_dim,)) img = model(noise) return Model(noise, img) def build_critic(self): model = Sequential() model.add(Conv2D(16, kernel_size=3, strides=2, input_shape=self.img_shape, padding="same")) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Conv2D(32, kernel_size=3, strides=2, padding="same")) model.add(ZeroPadding2D(padding=((0,1),(0,1)))) model.add(BatchNormalization(momentum=0.8)) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Conv2D(64, kernel_size=3, strides=2, padding="same")) model.add(BatchNormalization(momentum=0.8)) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Conv2D(128, kernel_size=3, strides=1, padding="same")) model.add(BatchNormalization(momentum=0.8)) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(1)) model.summary() img = Input(shape=self.img_shape) validity = model(img) return Model(img, validity) def train(self, epochs, batch_size, sample_interval=50): # Load the dataset (X_train, _), (_, _) = mnist.load_data() # Rescale -1 to 1 X_train = (X_train.astype(np.float32) - 127.5) / 127.5 X_train = np.expand_dims(X_train, axis=3) # Adversarial ground truths valid = -np.ones((batch_size, 1)) fake = np.ones((batch_size, 1)) dummy = np.zeros((batch_size, 1)) # Dummy gt for gradient penalty for epoch in range(epochs): for _ in range(self.n_critic): # --------------------- # Train Discriminator # --------------------- # Select a random batch of images idx = np.random.randint(0, X_train.shape[0], batch_size) imgs = X_train[idx] # Sample generator input noise = np.random.normal(0, 1, (batch_size, self.latent_dim)) # Train the critic d_loss = self.critic_model.train_on_batch([imgs, noise], [valid, fake, dummy]) # --------------------- # Train Generator # --------------------- g_loss = self.generator_model.train_on_batch(noise, valid) # Plot the progress print ("%d [D loss: %f] [G loss: %f]" % (epoch, d_loss[0], g_loss)) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.sample_images(epoch) def sample_images(self, epoch): r, c = 5, 5 noise = np.random.normal(0, 1, (r * c, self.latent_dim)) gen_imgs = self.generator.predict(noise) # Rescale images 0 - 1 gen_imgs = 0.5 * gen_imgs + 0.5 fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): axs[i,j].imshow(gen_imgs[cnt, :,:,0], cmap='gray') axs[i,j].axis('off') cnt += 1 fig.savefig("images/mnist_%d.png" % epoch) plt.close() if __name__ == '__main__': wgan = WGANGP() wgan.train(epochs=30000, batch_size=32, sample_interval=100)