[ { "path": "COPYRIGHT", "content": "Copyright (C) 2017-2018, IBM Corp.\nCopyright (c) 2016 Nicholas Carlini\n\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n" }, { "path": "LICENSE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "README.md", "content": "**As requested by IBM, this repository is moved to https://github.com/IBM/ZOO-Attack, but we aim to keep both repositories synced up.** The code is released under Apache License v2.\n\nZOO: Zeroth Order Optimization based Black-box Attacks to Deep Neural Networks \n=====================================\n\nZOO is a **z**eroth **o**rder **o**ptimization based attack to attack deep\nneural networks (DNNs). We propose an effective black-box attack that only\nrequires access to the input (images) and the output (confidence scores) of a\ntargeted DNN. We formularize the attack as an optimization problem (similar as\nCarlini and Wagner's attack), and propose a new loss function suitable for the\nblack-box setting. We use zeroth order stochastic coordinate descent to\noptimize on the target DNN directly, along with dimension reduction,\nhierarchical attack and importance sampling techniques to make the attack\nefficient. No transferability or substitute model is required.\n\nThere are two variants of ZOO, ZOO-ADAM and ZOO-Newton, corresponding to\ndifferent solvers (ADAM and Newton) to find the best coordinate update.\nIn practice ZOO-ADAM usually works better with fine-tuned parameters,\nbut ZOO-Newton is more stable when close to the optimal solution.\n\nThe experiment code is based on Carlini and Wagner's L2 attack, with\nzeroth order optimizer added in `l2_attack_black.py`. The inception model\nis updated to a new version (`inception_v3_2016_08_28.tar.gz`), and \nan unified interface `test_all.py` is added.\n\nFor more details, please see our paper:\n\n[ZOO: Zeroth Order Optimization based Black-box Attacks to Deep Neural Networks without Training Substitute Models](https://arxiv.org/abs/1708.03999)\nby Pin-Yu Chen\\*, Huan Zhang\\*, Yash Sharma, Jinfeng Yi, Cho-Jui Hsieh\n\n\\* Equal contribution\n\n\nSetup and train models\n-------------------------------------\n\nThe code is tested with python3 and TensorFlow v1.2 and v1.3. The following\npackages are required:\n\n```\nsudo apt-get install python3-pip\nsudo pip3 install --upgrade pip\nsudo pip3 install pillow scipy numpy tensorflow-gpu keras h5py numba\n```\n\nPrepare the MNIST and CIFAR-10 data and models for attack:\n\n```\npython3 train_models.py\n```\n\nTo download the inception model:\n\n```\npython3 setup_inception.py\n```\n\nTo prepare the ImageNet dataset, download and unzip the following archive:\n\nhttp://download.huan-zhang.com/datasets/adv/img.tar.gz\n\n\nand put the `imgs` folder in `../imagenetdata`. This path can be changed\nin `setup_inception.py`.\n\nRun attacks\n--------------------------------------\n\nAn unified attack interface, `test_all.py` is provided. Run `python3 test_all.py -h`\nto get a list of arguments and help.\n\nThe following are some examples of attacks:\n\nRun ZOO black-box targeted attack, on the mnist dataset with 200 images, with\nZOO-ADAM solver, search for best regularization constant for 9 iterations, and\nsave attack images to folder `black_results`. To run on the CIFAR-10 dataset,\nreplace 'mnist' with 'cifar10'.\n\n```\npython3 test_all.py -a black -d mnist -n 200 --solver adam -b 9 -s \"black_results\"\n```\n\nRun Carlini and Wagner's white-box targeted attack, on the mnist dataset with\n200 images, using the Z (logits) value in objective (only available in\nwhite-box setting), search for best regularization constant for 9 iterations,\nand save attack images to folder `white_results`.\n\n```\npython3 test_all.py -a white -d mnist -n 200 --use_zvalue -b 9 -s \"white_results\"\n```\n\nRun ZOO black-box *untargeted* attack, on the imagenet dataset with 150 images, with ZOO-ADAM\nsolver, do not binary search the regularization parameter (i.e., search only 1\ntime), and set the initial regularization parameter to a fixed value (10.0). Use\nattack-space dimension reduction with image resizing, and reset ADAM states\nwhen the first attack is found. Run a maximum of 1500 iterations, and print\nout loss every 10 iterations. Save attack images to folder `imagenet_untargeted`.\n\n```\npython3 test_all.py --untargeted -a black -d imagenet -n 150 --solver adam -b 1 -c 10.0 --use_resize --reset_adam -m 1500 -p 10 -s \"imagenet_untargeted\"\n```\n\nRun ZOO black-box targeted attack, on the imagenet dataset, with the 69th image\nonly. Set the regularization parameter to 10.0 and do not binary search. Use\nattack-space dimension reduction and hierarchical attack with image resizing,\nand reset ADAM states when the first attack is found. Run a maximum of 20000\niterations, and print out loss every 10 iterations. Save attack images to\nfolder `imagenet_all_tricks_img69`.\n\n\n```\npython3 test_all.py -a black --solver adam -d imagenet -f 69 -n 1 -c 10.0 --use_resize --reset_adam -m 20000 -p 10 -s \"imagenet_all_tricks_img69\"\n```\n\nImportance sampling is on by default for ImageNet data, and can be turned off by\n`--uniform` option. To change the hierarchical attack dimension scheduling,\nchange `l2_attack_black.py`, near line 580.\n\n" }, { "path": "cifar_blackbox.py", "content": "## Copyright (C) IBM Corp, 2017-2018\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\n\nimport numpy as np\nfrom six.moves import xrange\n\nimport keras\nfrom keras import backend\nfrom keras.utils.np_utils import to_categorical\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Flatten, Activation, Dropout\nfrom keras.datasets import cifar10\nfrom keras.utils import np_utils\n\nimport tensorflow as tf\nfrom tensorflow.python.platform import app\nfrom tensorflow.python.platform import flags\n\nfrom cleverhans.utils_keras import cnn_model\nfrom cleverhans.utils_tf import model_train, model_eval, batch_eval, tf_model_load\nfrom cleverhans.attacks import FastGradientMethod\nfrom cleverhans.attacks_tf import jacobian_graph, jacobian_augmentation\nfrom cleverhans.utils_keras import KerasModelWrapper\n\nfrom setup_cifar import CIFARModel\n\nFLAGS = flags.FLAGS\n\ndef data_cifar10():\n \"\"\"\n Preprocess CIFAR10 dataset\n :return:\n \"\"\"\n\n # These values are specific to CIFAR10\n img_rows = 32\n img_cols = 32\n nb_classes = 10\n\n # the data, shuffled and split between train and test sets\n (X_train, y_train), (X_test, y_test) = cifar10.load_data()\n\n if keras.backend.image_dim_ordering() == 'th':\n X_train = X_train.reshape(X_train.shape[0], 3, img_rows, img_cols)\n X_test = X_test.reshape(X_test.shape[0], 3, img_rows, img_cols)\n else:\n X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 3)\n X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 3)\n X_train = X_train.astype('float32')\n X_test = X_test.astype('float32')\n X_train /= 255\n X_test /= 255\n print('X_train shape:', X_train.shape)\n print(X_train.shape[0], 'train samples')\n print(X_test.shape[0], 'test samples')\n\n # convert class vectors to binary class matrices\n Y_train = np_utils.to_categorical(y_train, nb_classes)\n Y_test = np_utils.to_categorical(y_test, nb_classes)\n return X_train, Y_train, X_test, Y_test\n\n\ndef setup_tutorial():\n \"\"\"\n Helper function to check correct configuration of tf and keras for tutorial\n :return: True if setup checks completed\n \"\"\"\n\n # Set TF random seed to improve reproducibility\n tf.set_random_seed(1234)\n\n if not hasattr(backend, \"tf\"):\n raise RuntimeError(\"This tutorial requires keras to be configured\"\n \" to use the TensorFlow backend.\")\n\n # Image dimensions ordering should follow the Theano convention\n if keras.backend.image_dim_ordering() != 'tf':\n keras.backend.set_image_dim_ordering('tf')\n print(\"INFO: '~/.keras/keras.json' sets 'image_dim_ordering' \"\n \"to 'th', temporarily setting to 'tf'\")\n\n return True\n\n\ndef prep_bbox(sess, x, y, X_train, Y_train, X_test, Y_test,\n nb_epochs, batch_size, learning_rate):\n \"\"\"\n Define and train a model that simulates the \"remote\"\n black-box oracle described in the original paper.\n :param sess: the TF session\n :param x: the input placeholder for CIFAR\n :param y: the ouput placeholder for CIFAR\n :param X_train: the training data for the oracle\n :param Y_train: the training labels for the oracle\n :param X_test: the testing data for the oracle\n :param Y_test: the testing labels for the oracle\n :param nb_epochs: number of epochs to train model\n :param batch_size: size of training batches\n :param learning_rate: learning rate for training\n :return:\n \"\"\"\n\n # Define TF model graph (for the black-box model)\n model = CIFARModel(use_log = True).model\n # model = CIFARModel(use_log = True).model\n predictions = model(x)\n print(\"Defined TensorFlow model graph.\")\n\n # Train an CIFAR model\n if FLAGS.load_pretrain:\n # use the restored CIFAR model\n tf_model_load(sess)\n else:\n train_params = {\n 'nb_epochs': nb_epochs,\n 'batch_size': batch_size,\n 'learning_rate': learning_rate\n }\n model_train(sess, x, y, predictions, X_train, Y_train, verbose=True, save=True,\n args=train_params)\n \n # Print out the accuracy on legitimate data\n eval_params = {'batch_size': batch_size}\n accuracy = model_eval(sess, x, y, predictions, X_test, Y_test,\n args=eval_params)\n print('Test accuracy of black-box on legitimate test '\n 'examples: ' + str(accuracy))\n\n return model, predictions, accuracy\n\n\n\ndef train_sub(sess, x, y, bbox_preds, X_sub, Y_sub, nb_classes,\n nb_epochs_s, batch_size, learning_rate, data_aug, lmbda):\n \"\"\"\n This function creates the substitute by alternatively\n augmenting the training data and training the substitute.\n :param sess: TF session\n :param x: input TF placeholder\n :param y: output TF placeholder\n :param bbox_preds: output of black-box model predictions\n :param X_sub: initial substitute training data\n :param Y_sub: initial substitute training labels\n :param nb_classes: number of output classes\n :param nb_epochs_s: number of epochs to train substitute model\n :param batch_size: size of training batches\n :param learning_rate: learning rate for training\n :param data_aug: number of times substitute training data is augmented\n :param lmbda: lambda from arxiv.org/abs/1602.02697\n :return:\n \"\"\"\n # Define TF model graph (for the black-box model)\n model_sub = CIFARModel(use_log = True).model\n preds_sub = model_sub(x)\n print(\"Defined TensorFlow model graph for the substitute.\")\n\n # Define the Jacobian symbolically using TensorFlow\n grads = jacobian_graph(preds_sub, x, nb_classes)\n\n # Train the substitute and augment dataset alternatively\n for rho in xrange(data_aug):\n print(\"Substitute training epoch #\" + str(rho))\n train_params = {\n 'nb_epochs': nb_epochs_s,\n 'batch_size': batch_size,\n 'learning_rate': learning_rate\n }\n model_train(sess, x, y, preds_sub, X_sub, to_categorical(Y_sub),\n init_all=False, verbose=False, args=train_params)\n\n # If we are not at last substitute training iteration, augment dataset\n if rho < data_aug - 1:\n print(\"Augmenting substitute training data.\")\n # Perform the Jacobian augmentation\n X_sub = jacobian_augmentation(sess, x, X_sub, Y_sub, grads, lmbda)\n\n print(\"Labeling substitute training data.\")\n # Label the newly generated synthetic points using the black-box\n Y_sub = np.hstack([Y_sub, Y_sub])\n X_sub_prev = X_sub[int(len(X_sub)/2):]\n eval_params = {'batch_size': batch_size}\n bbox_val = batch_eval(sess, [x], [bbox_preds], [X_sub_prev],\n args=eval_params)[0]\n # Note here that we take the argmax because the adversary\n # only has access to the label (not the probabilities) output\n # by the black-box model\n Y_sub[int(len(X_sub)/2):] = np.argmax(bbox_val, axis=1)\n\n return model_sub, preds_sub\n\n\ndef cifar_blackbox(train_start=0, train_end=60000, test_start=0,\n test_end=10000, nb_classes=10, batch_size=128,\n learning_rate=0.001, nb_epochs=50, holdout=150, data_aug=6,\n nb_epochs_s=50, lmbda=0.1):\n \"\"\"\n CIFAR tutorial for the black-box attack from arxiv.org/abs/1602.02697\n :param train_start: index of first training set example\n :param train_end: index of last training set example\n :param test_start: index of first test set example\n :param test_end: index of last test set example\n :return: a dictionary with:\n * black-box model accuracy on test set\n * substitute model accuracy on test set\n * black-box model accuracy on adversarial examples transferred\n from the substitute model\n \"\"\"\n keras.layers.core.K.set_learning_phase(0)\n\n # Dictionary used to keep track and return key accuracies\n accuracies = {}\n\n # Perform tutorial setup\n assert setup_tutorial()\n\n # Create TF session and set as Keras backend session\n gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.3)\n sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))\n keras.backend.set_session(sess)\n\n # Get CIFAR data\n X_train, Y_train, X_test, Y_test = data_cifar10()\n\n # Initialize substitute training set reserved for adversary\n X_sub = X_test[:holdout]\n Y_sub = np.argmax(Y_test[:holdout], axis=1)\n\n # Redefine test set as remaining samples unavailable to adversaries\n X_test = X_test[holdout:]\n Y_test = Y_test[holdout:]\n\n # Define input and output TF placeholders\n x = tf.placeholder(tf.float32, shape=(None, 32, 32, 3))\n y = tf.placeholder(tf.float32, shape=(None, 10))\n\n # Simulate the black-box model locally\n # You could replace this by a remote labeling API for instance\n print(\"Preparing the black-box model.\")\n prep_bbox_out = prep_bbox(sess, x, y, X_train, Y_train, X_test, Y_test,\n nb_epochs, batch_size, learning_rate)\n model, bbox_preds, accuracies['bbox'] = prep_bbox_out\n\n # Train substitute using method from https://arxiv.org/abs/1602.02697\n print(\"Training the substitute model.\")\n train_sub_out = train_sub(sess, x, y, bbox_preds, X_sub, Y_sub,\n nb_classes, nb_epochs_s, batch_size,\n learning_rate, data_aug, lmbda)\n model_sub, preds_sub = train_sub_out\n\n # Evaluate the substitute model on clean test examples\n eval_params = {'batch_size': batch_size}\n acc = model_eval(sess, x, y, preds_sub, X_test, Y_test, args=eval_params)\n accuracies['sub'] = acc\n print('substitution model accuracy:', acc)\n\n # Find the correctly predicted labels\n original_predict = batch_eval(sess, [x], [bbox_preds], [X_test],\n args=eval_params)[0]\n original_class = np.argmax(original_predict, axis = 1)\n true_class = np.argmax(Y_test, axis = 1)\n mask = true_class == original_class\n print(np.sum(mask), \"out of\", mask.size, \"are correct labeled,\", len(X_test[mask])) \n \n # Initialize the Fast Gradient Sign Method (FGSM) attack object.\n fgsm_par = {'eps': 0.4, 'ord': np.inf, 'clip_min': 0., 'clip_max': 1.}\n wrap = KerasModelWrapper(model_sub)\n fgsm = FastGradientMethod(wrap, sess=sess)\n\n # Craft adversarial examples using the substitute\n eval_params = {'batch_size': batch_size}\n x_adv_sub = fgsm.generate(x, **fgsm_par)\n\n # Evaluate the accuracy of the \"black-box\" model on adversarial examples\n accuracy = model_eval(sess, x, y, model(x_adv_sub), X_test, Y_test,\n args=eval_params)\n print('Test accuracy of oracle on adversarial examples generated '\n 'using the substitute: ' + str(accuracy))\n accuracies['bbox_on_sub_adv_ex'] = accuracy\n \n # Evaluate the accuracy of the \"black-box\" model on adversarial examples\n accuracy = model_eval(sess, x, y, bbox_preds, X_test[mask], Y_test[mask],\n args=eval_params)\n print('Test accuracy of excluding originally incorrect labels: ' + str(accuracy))\n accuracies['bbox_on_sub_adv_ex_exc_ori'] = accuracy\n # Evaluate the accuracy of the \"black-box\" model on adversarial examples\n accuracy = model_eval(sess, x, y, model(x_adv_sub), X_test[mask], Y_test[mask],\n args=eval_params)\n print('Test accuracy of oracle on adversarial examples generated '\n 'using the substitute (excluding originally incorrect labels): ' + str(accuracy))\n accuracies['bbox_on_sub_adv_ex_exc'] = accuracy\n\n return accuracies\n\n\ndef main(argv=None):\n print(cifar_blackbox(nb_classes=FLAGS.nb_classes, batch_size=FLAGS.batch_size,\n learning_rate=FLAGS.learning_rate,\n nb_epochs=FLAGS.nb_epochs, holdout=FLAGS.holdout,\n data_aug=FLAGS.data_aug, nb_epochs_s=FLAGS.nb_epochs_s,\n lmbda=FLAGS.lmbda))\n\n\nif __name__ == '__main__':\n # General flags\n flags.DEFINE_integer('nb_classes', 10, 'Number of classes in problem')\n flags.DEFINE_integer('batch_size', 128, 'Size of training batches')\n flags.DEFINE_float('learning_rate', 0.0005, 'Learning rate for training')\n\n # Flags related to oracle\n flags.DEFINE_integer('nb_epochs', 50, 'Number of epochs to train model')\n\n # Flags related to substitute\n flags.DEFINE_integer('holdout', 150, 'Test set holdout for adversary')\n flags.DEFINE_integer('data_aug', 6, 'Nb of substitute data augmentations')\n flags.DEFINE_integer('nb_epochs_s', 50, 'Training epochs for substitute')\n flags.DEFINE_float('lmbda', 0.1, 'Lambda from arxiv.org/abs/1602.02697')\n\n # Flags related to saving/loading\n flags.DEFINE_bool('load_pretrain', False, 'load pretrained model from sub_saved/cifar-model')\n flags.DEFINE_string('train_dir', 'sub_saved', 'model saving path')\n flags.DEFINE_string('filename', 'cifar-model', 'cifar model name')\n app.run()\n" }, { "path": "l0_attack.py", "content": "## l0_attack.py -- attack a network optimizing for l_0 distance\n##\n## Copyright (C) IBM Corp, 2017-2018\n## Copyright (C) 2016, Nicholas Carlini .\n##\n## This program is licenced under the BSD 2-Clause licence,\n## contained in the LICENCE file in this directory.\n\nimport sys\nimport tensorflow as tf\nimport numpy as np\n\nMAX_ITERATIONS = 1000 # number of iterations to perform gradient descent\nABORT_EARLY = True # abort gradient descent upon first valid solution\nLEARNING_RATE = 1e-2 # larger values converge faster to less accurate results\nINITIAL_CONST = 1e-3 # the first value of c to start at\nLARGEST_CONST = 2e6 # the largest value of c to go up to before giving up\nREDUCE_CONST = False # try to lower c each iteration; faster to set to false\nTARGETED = True # should we target one specific class? or just be wrong?\nCONST_FACTOR = 2.0 # f>1, rate at which we increase constant, smaller better\n\nclass CarliniL0:\n def __init__(self, sess, model,\n targeted = TARGETED, learning_rate = LEARNING_RATE,\n max_iterations = MAX_ITERATIONS, abort_early = ABORT_EARLY,\n initial_const = INITIAL_CONST, largest_const = LARGEST_CONST,\n reduce_const = REDUCE_CONST, const_factor = CONST_FACTOR,\n independent_channels = False):\n \"\"\"\n The L_0 optimized attack. \n\n Returns adversarial examples for the supplied model.\n\n targeted: True if we should perform a targetted attack, False otherwise.\n learning_rate: The learning rate for the attack algorithm. Smaller values\n produce better results but are slower to converge.\n max_iterations: The maximum number of iterations. Larger values are more\n accurate; setting too small will require a large learning rate and will\n produce poor results.\n abort_early: If true, allows early aborts if gradient descent gets stuck.\n initial_const: The initial tradeoff-constant to use to tune the relative\n importance of distance and confidence. Should be set to a very small\n value (but positive).\n largest_const: The largest constant to use until we report failure. Should\n be set to a very large value.\n const_factor: The rate at which we should increase the constant, when the\n previous constant failed. Should be greater than one, smaller is better.\n independent_channels: set to false optimizes for number of pixels changed,\n set to true (not recommended) returns number of channels changed.\n \"\"\"\n\n self.model = model\n self.sess = sess\n\n self.TARGETED = targeted\n self.LEARNING_RATE = learning_rate\n self.MAX_ITERATIONS = max_iterations\n self.ABORT_EARLY = abort_early\n self.INITIAL_CONST = initial_const\n self.LARGEST_CONST = largest_const\n self.REDUCE_CONST = reduce_const\n self.const_factor = const_factor\n self.independent_channels = independent_channels\n\n self.grad = self.gradient_descent(sess, model)\n\n def gradient_descent(self, sess, model):\n def compare(x,y):\n if self.TARGETED:\n return x == y\n else:\n return x != y\n shape = (1,model.image_size,model.image_size,model.num_channels)\n \n # the variable to optimize over\n modifier = tf.Variable(np.zeros(shape,dtype=np.float32))\n\n # the variables we're going to hold, use for efficiency\n canchange = tf.Variable(np.zeros(shape),dtype=np.float32)\n simg = tf.Variable(np.zeros(shape,dtype=np.float32))\n original = tf.Variable(np.zeros(shape,dtype=np.float32))\n timg = tf.Variable(np.zeros(shape,dtype=np.float32))\n tlab = tf.Variable(np.zeros((1,model.num_labels),dtype=np.float32))\n const = tf.placeholder(tf.float32, [])\n\n # and the assignment to set the variables\n assign_modifier = tf.placeholder(np.float32,shape)\n assign_canchange = tf.placeholder(np.float32,shape)\n assign_simg = tf.placeholder(np.float32,shape)\n assign_original = tf.placeholder(np.float32,shape)\n assign_timg = tf.placeholder(np.float32,shape)\n assign_tlab = tf.placeholder(np.float32,(1,self.model.num_labels))\n\n # these are the variables to initialize when we run\n set_modifier = tf.assign(modifier, assign_modifier)\n setup = []\n setup.append(tf.assign(canchange, assign_canchange))\n setup.append(tf.assign(timg, assign_timg))\n setup.append(tf.assign(original, assign_original))\n setup.append(tf.assign(simg, assign_simg))\n setup.append(tf.assign(tlab, assign_tlab))\n \n newimg = (tf.tanh(modifier + simg)/2)*canchange+(1-canchange)*original\n \n output = model.predict(newimg)\n \n real = tf.reduce_sum((tlab)*output,1)\n other = tf.reduce_max((1-tlab)*output - (tlab*10000),1)\n if self.TARGETED:\n # if targetted, optimize for making the other class most likely\n loss1 = tf.maximum(0.0, other-real+.01)\n else:\n # if untargeted, optimize for making this class least likely.\n loss1 = tf.maximum(0.0, real-other+.01)\n\n # sum up the losses\n loss2 = tf.reduce_sum(tf.square(newimg-tf.tanh(timg)/2))\n loss = const*loss1+loss2\n \n outgrad = tf.gradients(loss, [modifier])[0]\n \n # setup the adam optimizer and keep track of variables we're creating\n start_vars = set(x.name for x in tf.global_variables())\n optimizer = tf.train.AdamOptimizer(self.LEARNING_RATE)\n train = optimizer.minimize(loss, var_list=[modifier])\n\n end_vars = tf.global_variables()\n new_vars = [x for x in end_vars if x.name not in start_vars]\n init = tf.variables_initializer(var_list=[modifier,canchange,simg,\n original,timg,tlab]+new_vars)\n\n \n def doit(oimgs, labs, starts, valid, CONST):\n # convert to tanh-space\n imgs = np.arctanh(np.array(oimgs)*1.999999)\n starts = np.arctanh(np.array(starts)*1.999999)\n\n # initialize the variables\n sess.run(init)\n sess.run(setup, {assign_timg: imgs, \n assign_tlab:labs, \n assign_simg: starts, \n assign_original: oimgs,\n assign_canchange: valid})\n\n while CONST < self.LARGEST_CONST:\n # try solving for each value of the constant\n print('try const', CONST)\n for step in range(self.MAX_ITERATIONS):\n feed_dict={const: CONST}\n\n # remember the old value\n oldmodifier = self.sess.run(modifier)\n\n if step%(self.MAX_ITERATIONS//10) == 0:\n print(step,*sess.run((loss1,loss2),feed_dict=feed_dict))\n\n # perform the update step\n _, works = sess.run([train, loss1], feed_dict=feed_dict)\n \n if works < .0001 and (self.ABORT_EARLY or step == CONST-1):\n # it worked previously, restore the old value and finish\n self.sess.run(set_modifier, {assign_modifier: oldmodifier})\n grads, scores, nimg = sess.run((outgrad, output,newimg),\n feed_dict=feed_dict)\n l2s=np.square(nimg-np.tanh(imgs)/2).sum(axis=(1,2,3))\n return grads, scores, nimg, CONST\n\n # we didn't succeed, increase constant and try again\n CONST *= self.const_factor\n return doit\n \n def attack(self, imgs, targets):\n \"\"\"\n Perform the L_0 attack on the given images for the given targets.\n\n If self.targeted is true, then the targets represents the target labels.\n If self.targeted is false, then targets are the original class labels.\n \"\"\"\n r = []\n for i,(img,target) in enumerate(zip(imgs, targets)):\n print(\"Attack iteration\",i)\n r.extend(self.attack_single(img, target))\n return np.array(r)\n\n def attack_single(self, img, target):\n \"\"\"\n Run the attack on a single image and label\n \"\"\"\n\n # the pixels we can change\n valid = np.ones((1,self.model.image_size,self.model.image_size,self.model.num_channels))\n\n # the previous image\n prev = np.copy(img).reshape((1,self.model.image_size,self.model.image_size,\n self.model.num_channels))\n last_solution = None\n const = self.INITIAL_CONST\n \n while True:\n # try to solve given this valid map\n res = self.grad([np.copy(img)], [target], np.copy(prev), \n valid, const)\n if res == None:\n # the attack failed, we return this as our final answer\n print(\"Final answer\",equal_count)\n return last_solution\n \n # the attack succeeded, now we pick new pixels to set to 0\n restarted = False\n gradientnorm, scores, nimg, const = res\n if self.REDUCE_CONST: const /= 2\n \n equal_count = self.model.image_size**2-np.sum(np.all(np.abs(img-nimg[0])<.0001,axis=2))\n print(\"Forced equal:\",np.sum(1-valid),\n \"Equal count:\",equal_count)\n if np.sum(valid) == 0:\n # if no pixels changed, return \n return [img]\n \n if self.independent_channels:\n # we are allowed to change each channel independently\n valid = valid.flatten()\n totalchange = abs(nimg[0]-img)*np.abs(gradientnorm[0])\n else:\n # we care only about which pixels change, not channels independently\n # compute total change as sum of change for each channel\n valid = valid.reshape((self.model.image_size**2,self.model.num_channels))\n totalchange = abs(np.sum(nimg[0]-img,axis=2))*np.sum(np.abs(gradientnorm[0]),axis=2)\n totalchange = totalchange.flatten()\n\n # set some of the pixels to 0 depending on their total change\n did = 0\n for e in np.argsort(totalchange):\n if np.all(valid[e]):\n did += 1\n valid[e] = 0\n\n if totalchange[e] > .01:\n # if this pixel changed a lot, skip\n break\n if did >= .3*equal_count**.5:\n # if we changed too many pixels, skip\n break\n\n valid = np.reshape(valid,(1,self.model.image_size,self.model.image_size,-1))\n print(\"Now forced equal:\",np.sum(1-valid))\n \n last_solution = prev = nimg\n" }, { "path": "l2_attack.py", "content": "## l2_attack.py -- attack a network optimizing for l_2 distance\n##\n## Copyright (C) IBM Corp, 2017-2018\n## Copyright (C) 2016, Nicholas Carlini .\n##\n## This program is licenced under the BSD 2-Clause licence,\n## contained in the LICENCE file in this directory.\n\nimport sys\nimport tensorflow as tf\nimport numpy as np\nimport time\n\nBINARY_SEARCH_STEPS = 1 # number of times to adjust the constant with binary search\nMAX_ITERATIONS = 10000 # number of iterations to perform gradient descent\nABORT_EARLY = True # if we stop improving, abort gradient descent early\nLEARNING_RATE = 2e-3 # larger values converge faster to less accurate results\nTARGETED = True # should we target one specific class? or just be wrong?\nCONFIDENCE = 0 # how strong the adversarial example should be\nINITIAL_CONST = 0.01 # the initial constant c to pick as a first guess\n\nclass CarliniL2:\n def __init__(self, sess, model, batch_size=1, confidence = CONFIDENCE,\n targeted = TARGETED, learning_rate = LEARNING_RATE,\n binary_search_steps = BINARY_SEARCH_STEPS, max_iterations = MAX_ITERATIONS, print_every = 100, early_stop_iters = 0,\n abort_early = ABORT_EARLY, \n initial_const = INITIAL_CONST,\n use_log = False, adam_beta1 = 0.9, adam_beta2 = 0.999):\n \"\"\"\n The L_2 optimized attack. \n\n This attack is the most efficient and should be used as the primary \n attack to evaluate potential defenses.\n\n Returns adversarial examples for the supplied model.\n\n confidence: Confidence of adversarial examples: higher produces examples\n that are farther away, but more strongly classified as adversarial.\n batch_size: Number of attacks to run simultaneously.\n targeted: True if we should perform a targetted attack, False otherwise.\n learning_rate: The learning rate for the attack algorithm. Smaller values\n produce better results but are slower to converge.\n binary_search_steps: The number of times we perform binary search to\n find the optimal tradeoff-constant between distance and confidence. \n max_iterations: The maximum number of iterations. Larger values are more\n accurate; setting too small will require a large learning rate and will\n produce poor results.\n abort_early: If true, allows early aborts if gradient descent gets stuck.\n initial_const: The initial tradeoff-constant to use to tune the relative\n importance of distance and confidence. If binary_search_steps is large,\n the initial constant is not important.\n \"\"\"\n\n image_size, num_channels, num_labels = model.image_size, model.num_channels, model.num_labels\n self.sess = sess\n self.TARGETED = targeted\n self.LEARNING_RATE = learning_rate\n self.MAX_ITERATIONS = max_iterations\n self.print_every = print_every\n self.early_stop_iters = early_stop_iters if early_stop_iters != 0 else max_iterations // 10\n print(\"early stop:\", self.early_stop_iters)\n self.BINARY_SEARCH_STEPS = binary_search_steps\n self.ABORT_EARLY = abort_early\n self.CONFIDENCE = confidence\n self.initial_const = initial_const\n self.batch_size = batch_size\n\n self.repeat = binary_search_steps >= 10\n\n shape = (batch_size,image_size,image_size,num_channels)\n \n # the variable we're going to optimize over\n self.modifier = tf.Variable(np.zeros(shape,dtype=np.float32))\n # self.modifier = tf.Variable(np.load('black_iter_350.npy').astype(np.float32).reshape(shape))\n\n # these are variables to be more efficient in sending data to tf\n self.timg = tf.Variable(np.zeros(shape), dtype=tf.float32)\n self.tlab = tf.Variable(np.zeros((batch_size,num_labels)), dtype=tf.float32)\n self.const = tf.Variable(np.zeros(batch_size), dtype=tf.float32)\n\n # and here's what we use to assign them\n self.assign_timg = tf.placeholder(tf.float32, shape)\n self.assign_tlab = tf.placeholder(tf.float32, (batch_size,num_labels))\n self.assign_const = tf.placeholder(tf.float32, [batch_size])\n \n # the resulting image, tanh'd to keep bounded from -0.5 to 0.5\n self.newimg = tf.tanh(self.modifier + self.timg)/2\n \n # prediction BEFORE-SOFTMAX of the model\n self.output = model.predict(self.newimg)\n \n # distance to the input data\n self.l2dist = tf.reduce_sum(tf.square(self.newimg-tf.tanh(self.timg)/2),[1,2,3])\n \n # compute the probability of the label class versus the maximum other\n self.real = tf.reduce_sum((self.tlab)*self.output,1)\n self.other = tf.reduce_max((1-self.tlab)*self.output - (self.tlab*10000),1)\n\n if self.TARGETED:\n if use_log:\n # loss1 = tf.maximum(- tf.log(self.other), - tf.log(self.real))\n # loss1 = - tf.log(self.real)\n loss1 = tf.maximum(0.0, tf.log(self.other + 1e-30) - tf.log(self.real + 1e-30))\n else:\n # if targetted, optimize for making the other class most likely\n loss1 = tf.maximum(0.0, self.other-self.real+self.CONFIDENCE)\n else:\n if use_log:\n # loss1 = tf.log(self.real)\n loss1 = tf.maximum(0.0, tf.log(self.real + 1e-30) - tf.log(self.other + 1e-30))\n else:\n # if untargeted, optimize for making this class least likely.\n loss1 = tf.maximum(0.0, self.real-self.other+self.CONFIDENCE)\n\n # sum up the losses\n self.loss2 = tf.reduce_sum(self.l2dist)\n self.loss1 = tf.reduce_sum(self.const*loss1)\n self.loss = self.loss1+self.loss2\n \n # Setup the adam optimizer and keep track of variables we're creating\n start_vars = set(x.name for x in tf.global_variables())\n # optimizer = tf.train.GradientDescentOptimizer(self.LEARNING_RATE)\n # optimizer = tf.train.MomentumOptimizer(self.LEARNING_RATE, 0.99)\n # optimizer = tf.train.RMSPropOptimizer(self.LEARNING_RATE)\n # optimizer = tf.train.AdadeltaOptimizer(self.LEARNING_RATE)\n optimizer = tf.train.AdamOptimizer(self.LEARNING_RATE, adam_beta1, adam_beta2)\n self.train = optimizer.minimize(self.loss, var_list=[self.modifier])\n end_vars = tf.global_variables()\n new_vars = [x for x in end_vars if x.name not in start_vars]\n\n # these are the variables to initialize when we run\n self.setup = []\n self.setup.append(self.timg.assign(self.assign_timg))\n self.setup.append(self.tlab.assign(self.assign_tlab))\n self.setup.append(self.const.assign(self.assign_const))\n # self.grad_op = tf.gradients(self.loss, self.modifier)\n \n self.init = tf.variables_initializer(var_list=[self.modifier]+new_vars)\n\n def attack(self, imgs, targets):\n \"\"\"\n Perform the L_2 attack on the given images for the given targets.\n\n If self.targeted is true, then the targets represents the target labels.\n If self.targeted is false, then targets are the original class labels.\n \"\"\"\n r = []\n print('go up to',len(imgs))\n for i in range(0,len(imgs),self.batch_size):\n print('tick',i)\n r.extend(self.attack_batch(imgs[i:i+self.batch_size], targets[i:i+self.batch_size])[0])\n return np.array(r)\n\n def attack_batch(self, imgs, labs):\n \"\"\"\n Run the attack on a batch of images and labels.\n \"\"\"\n def compare(x,y):\n if not isinstance(x, (float, int, np.int64)):\n x = np.copy(x)\n if self.TARGETED:\n x[y] -= self.CONFIDENCE\n else:\n x[y] += self.CONFIDENCE\n x = np.argmax(x)\n if self.TARGETED:\n return x == y\n else:\n return x != y\n\n batch_size = self.batch_size\n\n # convert to tanh-space\n imgs = np.arctanh(imgs*1.999999)\n\n # set the lower and upper bounds accordingly\n lower_bound = np.zeros(batch_size)\n CONST = np.ones(batch_size)*self.initial_const\n upper_bound = np.ones(batch_size)*1e10\n\n # the best l2, score, and image attack\n o_bestl2 = [1e10]*batch_size\n o_bestscore = [-1]*batch_size\n o_bestattack = [np.zeros(imgs[0].shape)]*batch_size\n o_best_const = [self.initial_const]*batch_size\n \n for outer_step in range(self.BINARY_SEARCH_STEPS):\n print(\"current best l2\", o_bestl2)\n # completely reset adam's internal state.\n self.sess.run(self.init)\n batch = imgs[:batch_size]\n batchlab = labs[:batch_size]\n \n bestl2 = [1e10]*batch_size\n bestscore = [-1]*batch_size\n\n # The last iteration (if we run many steps) repeat the search once.\n if self.repeat == True and outer_step == self.BINARY_SEARCH_STEPS-1:\n CONST = upper_bound\n\n # set the variables so that we don't have to send them over again\n self.sess.run(self.setup, {self.assign_timg: batch,\n self.assign_tlab: batchlab,\n self.assign_const: CONST})\n \n prev = 1e6\n train_timer = 0.0\n for iteration in range(self.MAX_ITERATIONS):\n # print out the losses every 10%\n if iteration%(self.MAX_ITERATIONS//self.print_every) == 0:\n # print(iteration,self.sess.run((self.loss,self.real,self.other,self.loss1,self.loss2)))\n # grad = self.sess.run(self.grad_op)\n # old_modifier = self.sess.run(self.modifier)\n # np.save('white_iter_{}'.format(iteration), modifier)\n loss, real, other, loss1, loss2 = self.sess.run((self.loss,self.real,self.other,self.loss1,self.loss2))\n print(\"[STATS][L2] iter = {}, time = {:.3f}, loss = {:.5g}, real = {:.5g}, other = {:.5g}, loss1 = {:.5g}, loss2 = {:.5g}\".format(iteration, train_timer, loss, real[0], other[0], loss1, loss2))\n sys.stdout.flush()\n\n attack_begin_time = time.time()\n # perform the attack \n _, l, l2s, scores, nimg = self.sess.run([self.train, self.loss, \n self.l2dist, self.output, \n self.newimg])\n\n new_modifier = self.sess.run(self.modifier)\n \n # print(grad[0].reshape(-1))\n # print((old_modifier - new_modifier).reshape(-1))\n\n # check if we should abort search if we're getting nowhere.\n if self.ABORT_EARLY and iteration % self.early_stop_iters == 0:\n if l > prev*.9999:\n print(\"Early stopping because there is no improvement\")\n break\n prev = l\n\n # adjust the best result found so far\n for e,(l2,sc,ii) in enumerate(zip(l2s,scores,nimg)):\n if l2 < bestl2[e] and compare(sc, np.argmax(batchlab[e])):\n bestl2[e] = l2\n bestscore[e] = np.argmax(sc)\n if l2 < o_bestl2[e] and compare(sc, np.argmax(batchlab[e])):\n o_bestl2[e] = l2\n o_bestscore[e] = np.argmax(sc)\n o_bestattack[e] = ii\n o_best_const[e] = CONST[e]\n\n train_timer += time.time() - attack_begin_time\n\n # adjust the constant as needed\n for e in range(batch_size):\n if compare(bestscore[e], np.argmax(batchlab[e])) and bestscore[e] != -1:\n # modifier = self.sess.run(self.modifier)\n # np.save(\"best.model\", modifier)\n print('old constant: ', CONST[e])\n # success, divide const by two\n upper_bound[e] = min(upper_bound[e],CONST[e])\n if upper_bound[e] < 1e9:\n CONST[e] = (lower_bound[e] + upper_bound[e])/2\n print('new constant: ', CONST[e])\n else:\n print('old constant: ', CONST[e])\n # failure, either multiply by 10 if no solution found yet\n # or do binary search with the known upper bound\n lower_bound[e] = max(lower_bound[e],CONST[e])\n if upper_bound[e] < 1e9:\n CONST[e] = (lower_bound[e] + upper_bound[e])/2\n else:\n CONST[e] *= 10\n print('new constant: ', CONST[e])\n\n # return the best solution found\n o_bestl2 = np.array(o_bestl2)\n return np.array(o_bestattack), o_best_const\n" }, { "path": "l2_attack_black.py", "content": "## l2_attack_black.py -- attack a black-box network optimizing for l_2 distance\n##\n## Copyright (C) IBM Corp, 2017-2018\n## Copyright (C) 2017, Huan Zhang .\n## Copyright (C) 2016, Nicholas Carlini .\n##\n## This program is licenced under the BSD 2-Clause licence,\n## contained in the LICENCE file in this directory.\n\nimport sys\nimport os\nimport tensorflow as tf\nimport numpy as np\nimport scipy.misc\nfrom numba import jit\nimport math\nimport time\n\nBINARY_SEARCH_STEPS = 1 # number of times to adjust the constant with binary search\nMAX_ITERATIONS = 10000 # number of iterations to perform gradient descent\nABORT_EARLY = True # if we stop improving, abort gradient descent early\nLEARNING_RATE = 2e-3 # larger values converge faster to less accurate results\nTARGETED = True # should we target one specific class? or just be wrong?\nCONFIDENCE = 0 # how strong the adversarial example should be\nINITIAL_CONST = 0.5 # the initial constant c to pick as a first guess\n\n@jit(nopython=True)\ndef coordinate_ADAM(losses, indice, grad, hess, batch_size, mt_arr, vt_arr, real_modifier, up, down, lr, adam_epoch, beta1, beta2, proj):\n # indice = np.array(range(0, 3*299*299), dtype = np.int32)\n for i in range(batch_size):\n grad[i] = (losses[i*2+1] - losses[i*2+2]) / 0.0002 \n # true_grads = self.sess.run(self.grad_op, feed_dict={self.modifier: self.real_modifier})\n # true_grads, losses, l2s, scores, nimgs = self.sess.run([self.grad_op, self.loss, self.l2dist, self.output, self.newimg], feed_dict={self.modifier: self.real_modifier})\n # grad = true_grads[0].reshape(-1)[indice]\n # print(grad, true_grads[0].reshape(-1)[indice])\n # self.real_modifier.reshape(-1)[indice] -= self.LEARNING_RATE * grad\n # self.real_modifier -= self.LEARNING_RATE * true_grads[0]\n # ADAM update\n mt = mt_arr[indice]\n mt = beta1 * mt + (1 - beta1) * grad\n mt_arr[indice] = mt\n vt = vt_arr[indice]\n vt = beta2 * vt + (1 - beta2) * (grad * grad)\n vt_arr[indice] = vt\n # epoch is an array; for each index we can have a different epoch number\n epoch = adam_epoch[indice]\n corr = (np.sqrt(1 - np.power(beta2,epoch))) / (1 - np.power(beta1, epoch))\n m = real_modifier.reshape(-1)\n old_val = m[indice] \n old_val -= lr * corr * mt / (np.sqrt(vt) + 1e-8)\n # set it back to [-0.5, +0.5] region\n if proj:\n old_val = np.maximum(np.minimum(old_val, up[indice]), down[indice])\n # print(grad)\n # print(old_val - m[indice])\n m[indice] = old_val\n adam_epoch[indice] = epoch + 1\n\n@jit(nopython=True)\ndef coordinate_Newton(losses, indice, grad, hess, batch_size, mt_arr, vt_arr, real_modifier, up, down, lr, adam_epoch, beta1, beta2, proj):\n # def sign(x):\n # return np.piecewise(x, [x < 0, x >= 0], [-1, 1])\n cur_loss = losses[0]\n for i in range(batch_size):\n grad[i] = (losses[i*2+1] - losses[i*2+2]) / 0.0002 \n hess[i] = (losses[i*2+1] - 2 * cur_loss + losses[i*2+2]) / (0.0001 * 0.0001)\n # print(\"New epoch:\")\n # print('grad', grad)\n # print('hess', hess)\n # hess[hess < 0] = 1.0\n # hess[np.abs(hess) < 0.1] = sign(hess[np.abs(hess) < 0.1]) * 0.1\n # negative hessian cannot provide second order information, just do a gradient descent\n hess[hess < 0] = 1.0\n # hessian too small, could be numerical problems\n hess[hess < 0.1] = 0.1\n # print(hess)\n m = real_modifier.reshape(-1)\n old_val = m[indice] \n old_val -= lr * grad / hess\n # set it back to [-0.5, +0.5] region\n if proj:\n old_val = np.maximum(np.minimum(old_val, up[indice]), down[indice])\n # print('delta', old_val - m[indice])\n m[indice] = old_val\n # print(m[indice])\n\n@jit(nopython=True)\ndef coordinate_Newton_ADAM(losses, indice, grad, hess, batch_size, mt_arr, vt_arr, real_modifier, up, down, lr, adam_epoch, beta1, beta2, proj):\n cur_loss = losses[0]\n for i in range(batch_size):\n grad[i] = (losses[i*2+1] - losses[i*2+2]) / 0.0002 \n hess[i] = (losses[i*2+1] - 2 * cur_loss + losses[i*2+2]) / (0.0001 * 0.0001)\n # print(\"New epoch:\")\n # print(grad)\n # print(hess)\n # positive hessian, using newton's method\n hess_indice = (hess >= 0)\n # print(hess_indice)\n # negative hessian, using ADAM\n adam_indice = (hess < 0)\n # print(adam_indice)\n # print(sum(hess_indice), sum(adam_indice))\n hess[hess < 0] = 1.0\n hess[hess < 0.1] = 0.1\n # hess[np.abs(hess) < 0.1] = sign(hess[np.abs(hess) < 0.1]) * 0.1\n # print(adam_indice)\n # Newton's Method\n m = real_modifier.reshape(-1)\n old_val = m[indice[hess_indice]] \n old_val -= lr * grad[hess_indice] / hess[hess_indice]\n # set it back to [-0.5, +0.5] region\n if proj:\n old_val = np.maximum(np.minimum(old_val, up[indice[hess_indice]]), down[indice[hess_indice]])\n m[indice[hess_indice]] = old_val\n # ADMM\n mt = mt_arr[indice]\n mt = beta1 * mt + (1 - beta1) * grad\n mt_arr[indice] = mt\n vt = vt_arr[indice]\n vt = beta2 * vt + (1 - beta2) * (grad * grad)\n vt_arr[indice] = vt\n # epoch is an array; for each index we can have a different epoch number\n epoch = adam_epoch[indice]\n corr = (np.sqrt(1 - np.power(beta2,epoch[adam_indice]))) / (1 - np.power(beta1, epoch[adam_indice]))\n old_val = m[indice[adam_indice]] \n old_val -= lr * corr * mt[adam_indice] / (np.sqrt(vt[adam_indice]) + 1e-8)\n # old_val -= lr * grad[adam_indice]\n # set it back to [-0.5, +0.5] region\n if proj:\n old_val = np.maximum(np.minimum(old_val, up[indice[adam_indice]]), down[indice[adam_indice]])\n m[indice[adam_indice]] = old_val\n adam_epoch[indice] = epoch + 1\n # print(m[indice])\n\nclass BlackBoxL2:\n def __init__(self, sess, model, batch_size=1, confidence = CONFIDENCE,\n targeted = TARGETED, learning_rate = LEARNING_RATE,\n binary_search_steps = BINARY_SEARCH_STEPS, max_iterations = MAX_ITERATIONS, print_every = 100, early_stop_iters = 0,\n abort_early = ABORT_EARLY, \n initial_const = INITIAL_CONST,\n use_log = False, use_tanh = True, use_resize = False, adam_beta1 = 0.9, adam_beta2 = 0.999, reset_adam_after_found = False,\n solver = \"adam\", save_ckpts = \"\", load_checkpoint = \"\", start_iter = 0,\n init_size = 32, use_importance = True):\n \"\"\"\n The L_2 optimized attack. \n\n This attack is the most efficient and should be used as the primary \n attack to evaluate potential defenses.\n\n Returns adversarial examples for the supplied model.\n\n confidence: Confidence of adversarial examples: higher produces examples\n that are farther away, but more strongly classified as adversarial.\n batch_size: Number of gradient evaluations to run simultaneously.\n targeted: True if we should perform a targetted attack, False otherwise.\n learning_rate: The learning rate for the attack algorithm. Smaller values\n produce better results but are slower to converge.\n binary_search_steps: The number of times we perform binary search to\n find the optimal tradeoff-constant between distance and confidence. \n max_iterations: The maximum number of iterations. Larger values are more\n accurate; setting too small will require a large learning rate and will\n produce poor results.\n abort_early: If true, allows early aborts if gradient descent gets stuck.\n initial_const: The initial tradeoff-constant to use to tune the relative\n importance of distance and confidence. If binary_search_steps is large,\n the initial constant is not important.\n \"\"\"\n\n image_size, num_channels, num_labels = model.image_size, model.num_channels, model.num_labels\n self.model = model\n self.sess = sess\n self.TARGETED = targeted\n self.LEARNING_RATE = learning_rate\n self.MAX_ITERATIONS = max_iterations\n self.print_every = print_every\n self.early_stop_iters = early_stop_iters if early_stop_iters != 0 else max_iterations // 10\n print(\"early stop:\", self.early_stop_iters)\n self.BINARY_SEARCH_STEPS = binary_search_steps\n self.ABORT_EARLY = abort_early\n self.CONFIDENCE = confidence\n self.initial_const = initial_const\n self.start_iter = start_iter\n self.batch_size = batch_size\n self.num_channels = num_channels\n self.resize_init_size = init_size\n self.use_importance = use_importance\n if use_resize:\n self.small_x = self.resize_init_size\n self.small_y = self.resize_init_size\n else:\n self.small_x = image_size\n self.small_y = image_size\n\n self.use_tanh = use_tanh\n self.use_resize = use_resize\n self.save_ckpts = save_ckpts\n if save_ckpts:\n os.system(\"mkdir -p {}\".format(save_ckpts))\n\n self.repeat = binary_search_steps >= 10\n\n # each batch has a different modifier value (see below) to evaluate\n # small_shape = (None,self.small_x,self.small_y,num_channels)\n shape = (None,image_size,image_size,num_channels)\n single_shape = (image_size, image_size, num_channels)\n small_single_shape = (self.small_x, self.small_y, num_channels)\n \n # the variable we're going to optimize over\n # support multiple batches\n # support any size image, will be resized to model native size\n if self.use_resize:\n self.modifier = tf.placeholder(tf.float32, shape=(None, None, None, None))\n # scaled up image\n self.scaled_modifier = tf.image.resize_images(self.modifier, [image_size, image_size])\n # operator used for resizing image\n self.resize_size_x = tf.placeholder(tf.int32)\n self.resize_size_y = tf.placeholder(tf.int32)\n self.resize_input = tf.placeholder(tf.float32, shape=(1, None, None, None))\n self.resize_op = tf.image.resize_images(self.resize_input, [self.resize_size_x, self.resize_size_y])\n else:\n self.modifier = tf.placeholder(tf.float32, shape=(None, image_size, image_size, num_channels))\n # no resize\n self.scaled_modifier = self.modifier\n # the real variable, initialized to 0\n self.load_checkpoint = load_checkpoint\n if load_checkpoint:\n # if checkpoint is incorrect reshape will fail\n print(\"Using checkpint\", load_checkpoint)\n self.real_modifier = np.load(load_checkpoint).reshape((1,) + small_single_shape)\n else:\n self.real_modifier = np.zeros((1,) + small_single_shape, dtype=np.float32)\n # self.real_modifier = np.random.randn(image_size * image_size * num_channels).astype(np.float32).reshape((1,) + single_shape)\n # self.real_modifier /= np.linalg.norm(self.real_modifier) \n # these are variables to be more efficient in sending data to tf\n # we only work on 1 image at once; the batch is for evaluation loss at different modifiers\n self.timg = tf.Variable(np.zeros(single_shape), dtype=tf.float32)\n self.tlab = tf.Variable(np.zeros(num_labels), dtype=tf.float32)\n self.const = tf.Variable(0.0, dtype=tf.float32)\n\n # and here's what we use to assign them\n self.assign_timg = tf.placeholder(tf.float32, single_shape)\n self.assign_tlab = tf.placeholder(tf.float32, num_labels)\n self.assign_const = tf.placeholder(tf.float32)\n \n # the resulting image, tanh'd to keep bounded from -0.5 to 0.5\n # broadcast self.timg to every dimension of modifier\n if use_tanh:\n self.newimg = tf.tanh(self.scaled_modifier + self.timg)/2\n else:\n self.newimg = self.scaled_modifier + self.timg\n \n # prediction BEFORE-SOFTMAX of the model\n # now we have output at #batch_size different modifiers\n # the output should have shape (batch_size, num_labels)\n self.output = model.predict(self.newimg)\n \n # distance to the input data\n if use_tanh:\n self.l2dist = tf.reduce_sum(tf.square(self.newimg-tf.tanh(self.timg)/2), [1,2,3])\n else:\n self.l2dist = tf.reduce_sum(tf.square(self.newimg - self.timg), [1,2,3])\n \n # compute the probability of the label class versus the maximum other\n # self.tlab * self.output selects the Z value of real class\n # because self.tlab is an one-hot vector\n # the reduce_sum removes extra zeros, now get a vector of size #batch_size\n self.real = tf.reduce_sum((self.tlab)*self.output,1)\n # (1-self.tlab)*self.output gets all Z values for other classes\n # Because soft Z values are negative, it is possible that all Z values are less than 0\n # and we mistakenly select the real class as the max. So we minus 10000 for real class\n self.other = tf.reduce_max((1-self.tlab)*self.output - (self.tlab*10000),1)\n\n # If self.targeted is true, then the targets represents the target labels.\n # If self.targeted is false, then targets are the original class labels.\n if self.TARGETED:\n if use_log:\n # loss1 = - tf.log(self.real)\n loss1 = tf.maximum(0.0, tf.log(self.other + 1e-30) - tf.log(self.real + 1e-30))\n else:\n # if targetted, optimize for making the other class (real) most likely\n loss1 = tf.maximum(0.0, self.other-self.real+self.CONFIDENCE)\n else:\n if use_log:\n # loss1 = tf.log(self.real)\n loss1 = tf.maximum(0.0, tf.log(self.real + 1e-30) - tf.log(self.other + 1e-30))\n else:\n # if untargeted, optimize for making this class least likely.\n loss1 = tf.maximum(0.0, self.real-self.other+self.CONFIDENCE)\n\n # sum up the losses (output is a vector of #batch_size)\n self.loss2 = self.l2dist\n self.loss1 = self.const*loss1\n self.loss = self.loss1+self.loss2\n \n # these are the variables to initialize when we run\n self.setup = []\n self.setup.append(self.timg.assign(self.assign_timg))\n self.setup.append(self.tlab.assign(self.assign_tlab))\n self.setup.append(self.const.assign(self.assign_const))\n\n # prepare the list of all valid variables\n var_size = self.small_x * self.small_y * num_channels\n self.use_var_len = var_size\n self.var_list = np.array(range(0, self.use_var_len), dtype = np.int32)\n self.used_var_list = np.zeros(var_size, dtype = np.int32)\n self.sample_prob = np.ones(var_size, dtype = np.float32) / var_size\n\n # upper and lower bounds for the modifier\n self.modifier_up = np.zeros(var_size, dtype = np.float32)\n self.modifier_down = np.zeros(var_size, dtype = np.float32)\n\n # random permutation for coordinate update\n self.perm = np.random.permutation(var_size)\n self.perm_index = 0\n\n # ADAM status\n self.mt = np.zeros(var_size, dtype = np.float32)\n self.vt = np.zeros(var_size, dtype = np.float32)\n # self.beta1 = 0.8\n # self.beta2 = 0.99\n self.beta1 = adam_beta1\n self.beta2 = adam_beta2\n self.reset_adam_after_found = reset_adam_after_found\n self.adam_epoch = np.ones(var_size, dtype = np.int32)\n self.stage = 0\n # variables used during optimization process\n self.grad = np.zeros(batch_size, dtype = np.float32)\n self.hess = np.zeros(batch_size, dtype = np.float32)\n # for testing\n self.grad_op = tf.gradients(self.loss, self.modifier)\n # compile numba function\n # self.coordinate_ADAM_numba = jit(coordinate_ADAM, nopython = True)\n # self.coordinate_ADAM_numba.recompile()\n # print(self.coordinate_ADAM_numba.inspect_llvm())\n # np.set_printoptions(threshold=np.nan)\n # set solver\n solver = solver.lower()\n self.solver_name = solver\n if solver == \"adam\":\n self.solver = coordinate_ADAM\n elif solver == \"newton\":\n self.solver = coordinate_Newton\n elif solver == \"adam_newton\":\n self.solver = coordinate_Newton_ADAM\n elif solver != \"fake_zero\":\n print(\"unknown solver\", solver)\n self.solver = coordinate_ADAM\n print(\"Using\", solver, \"solver\")\n\n def max_pooling(self, image, size):\n img_pool = np.copy(image)\n img_x = image.shape[0]\n img_y = image.shape[1]\n for i in range(0, img_x, size):\n for j in range(0, img_y, size):\n img_pool[i:i+size, j:j+size] = np.max(image[i:i+size, j:j+size])\n return img_pool\n\n def get_new_prob(self, prev_modifier, gen_double = False):\n prev_modifier = np.squeeze(prev_modifier)\n old_shape = prev_modifier.shape\n if gen_double:\n new_shape = (old_shape[0]*2, old_shape[1]*2, old_shape[2])\n else:\n new_shape = old_shape\n prob = np.empty(shape=new_shape, dtype = np.float32)\n for i in range(prev_modifier.shape[2]):\n image = np.abs(prev_modifier[:,:,i])\n image_pool = self.max_pooling(image, old_shape[0] // 8)\n if gen_double:\n prob[:,:,i] = scipy.misc.imresize(image_pool, 2.0, 'nearest', mode = 'F')\n else:\n prob[:,:,i] = image_pool\n prob /= np.sum(prob)\n return prob\n\n\n def resize_img(self, small_x, small_y, reset_only = False):\n self.small_x = small_x\n self.small_y = small_y\n small_single_shape = (self.small_x, self.small_y, self.num_channels)\n if reset_only:\n self.real_modifier = np.zeros((1,) + small_single_shape, dtype=np.float32)\n else:\n # run the resize_op once to get the scaled image\n prev_modifier = np.copy(self.real_modifier)\n self.real_modifier = self.sess.run(self.resize_op, feed_dict={self.resize_size_x: self.small_x, self.resize_size_y: self.small_y, self.resize_input: self.real_modifier})\n # prepare the list of all valid variables\n var_size = self.small_x * self.small_y * self.num_channels\n self.use_var_len = var_size\n self.var_list = np.array(range(0, self.use_var_len), dtype = np.int32)\n # ADAM status\n self.mt = np.zeros(var_size, dtype = np.float32)\n self.vt = np.zeros(var_size, dtype = np.float32)\n self.adam_epoch = np.ones(var_size, dtype = np.int32)\n # update sample probability\n if reset_only:\n self.sample_prob = np.ones(var_size, dtype = np.float32) / var_size\n else:\n self.sample_prob = self.get_new_prob(prev_modifier, True)\n self.sample_prob = self.sample_prob.reshape(var_size)\n\n def fake_blackbox_optimizer(self):\n true_grads, losses, l2s, loss1, loss2, scores, nimgs = self.sess.run([self.grad_op, self.loss, self.l2dist, self.loss1, self.loss2, self.output, self.newimg], feed_dict={self.modifier: self.real_modifier})\n # ADAM update\n grad = true_grads[0].reshape(-1)\n # print(true_grads[0])\n epoch = self.adam_epoch[0]\n mt = self.beta1 * self.mt + (1 - self.beta1) * grad\n vt = self.beta2 * self.vt + (1 - self.beta2) * np.square(grad)\n corr = (math.sqrt(1 - self.beta2 ** epoch)) / (1 - self.beta1 ** epoch)\n # print(grad.shape, mt.shape, vt.shape, self.real_modifier.shape)\n # m is a *view* of self.real_modifier\n m = self.real_modifier.reshape(-1)\n # this is in-place\n m -= self.LEARNING_RATE * corr * (mt / (np.sqrt(vt) + 1e-8))\n self.mt = mt\n self.vt = vt\n # m -= self.LEARNING_RATE * grad\n if not self.use_tanh:\n m_proj = np.maximum(np.minimum(m, self.modifier_up), self.modifier_down)\n np.copyto(m, m_proj)\n self.adam_epoch[0] = epoch + 1\n return losses[0], l2s[0], loss1[0], loss2[0], scores[0], nimgs[0]\n\n\n def blackbox_optimizer(self, iteration):\n # build new inputs, based on current variable value\n var = np.repeat(self.real_modifier, self.batch_size * 2 + 1, axis=0)\n var_size = self.real_modifier.size\n # print(s, \"variables remaining\")\n # var_indice = np.random.randint(0, self.var_list.size, size=self.batch_size)\n if self.use_importance:\n var_indice = np.random.choice(self.var_list.size, self.batch_size, replace=False, p = self.sample_prob)\n else:\n var_indice = np.random.choice(self.var_list.size, self.batch_size, replace=False)\n indice = self.var_list[var_indice]\n # indice = self.var_list\n # regenerate the permutations if we run out\n # if self.perm_index + self.batch_size >= var_size:\n # self.perm = np.random.permutation(var_size)\n # self.perm_index = 0\n # indice = self.perm[self.perm_index:self.perm_index + self.batch_size]\n # b[0] has the original modifier, b[1] has one index added 0.0001\n for i in range(self.batch_size):\n var[i * 2 + 1].reshape(-1)[indice[i]] += 0.0001\n var[i * 2 + 2].reshape(-1)[indice[i]] -= 0.0001\n losses, l2s, loss1, loss2, scores, nimgs = self.sess.run([self.loss, self.l2dist, self.loss1, self.loss2, self.output, self.newimg], feed_dict={self.modifier: var})\n # losses = self.sess.run(self.loss, feed_dict={self.modifier: var})\n # t_grad = self.sess.run(self.grad_op, feed_dict={self.modifier: self.real_modifier})\n # self.grad = t_grad[0].reshape(-1)\n # true_grads = self.sess.run(self.grad_op, feed_dict={self.modifier: self.real_modifier})\n # self.coordinate_ADAM_numba(losses, indice, self.grad, self.hess, self.batch_size, self.mt, self.vt, self.real_modifier, self.modifier_up, self.modifier_down, self.LEARNING_RATE, self.adam_epoch, self.beta1, self.beta2, not self.use_tanh)\n # coordinate_ADAM(losses, indice, self.grad, self.hess, self.batch_size, self.mt, self.vt, self.real_modifier, self.modifier_up, self.modifier_down, self.LEARNING_RATE, self.adam_epoch, self.beta1, self.beta2, not self.use_tanh)\n # coordinate_ADAM(losses, indice, self.grad, self.hess, self.batch_size, self.mt, self.vt, self.real_modifier, self.modifier_up, self.modifier_down, self.LEARNING_RATE, self.adam_epoch, self.beta1, self.beta2, not self.use_tanh, true_grads)\n # coordinate_Newton(losses, indice, self.grad, self.hess, self.batch_size, self.mt, self.vt, self.real_modifier, self.modifier_up, self.modifier_down, self.LEARNING_RATE, self.adam_epoch, self.beta1, self.beta2, not self.use_tanh)\n # coordinate_Newton_ADAM(losses, indice, self.grad, self.hess, self.batch_size, self.mt, self.vt, self.real_modifier, self.modifier_up, self.modifier_down, self.LEARNING_RATE, self.adam_epoch, self.beta1, self.beta2, not self.use_tanh)\n self.solver(losses, indice, self.grad, self.hess, self.batch_size, self.mt, self.vt, self.real_modifier, self.modifier_up, self.modifier_down, self.LEARNING_RATE, self.adam_epoch, self.beta1, self.beta2, not self.use_tanh)\n # adjust sample probability, sample around the points with large gradient\n if self.save_ckpts:\n np.save('{}/iter{}'.format(self.save_ckpts, iteration), self.real_modifier)\n\n if self.real_modifier.shape[0] > self.resize_init_size:\n self.sample_prob = self.get_new_prob(self.real_modifier)\n # self.sample_prob = self.get_new_prob(tmp_mt.reshape(self.real_modifier.shape))\n self.sample_prob = self.sample_prob.reshape(var_size)\n\n # if the gradient is too small, do not optimize on this variable\n # self.var_list = np.delete(self.var_list, indice[np.abs(self.grad) < 5e-3])\n # reset the list every 10000 iterations\n # if iteration%200 == 0:\n # print(\"{} variables remained at last stage\".format(self.var_list.size))\n # var_size = self.real_modifier.size\n # self.var_list = np.array(range(0, var_size))\n return losses[0], l2s[0], loss1[0], loss2[0], scores[0], nimgs[0]\n # return losses[0]\n\n def attack(self, imgs, targets):\n \"\"\"\n Perform the L_2 attack on the given images for the given targets.\n\n If self.targeted is true, then the targets represents the target labels.\n If self.targeted is false, then targets are the original class labels.\n \"\"\"\n r = []\n print('go up to',len(imgs))\n # we can only run 1 image at a time, minibatches are used for gradient evaluation\n for i in range(0,len(imgs)):\n print('tick',i)\n r.extend(self.attack_batch(imgs[i], targets[i]))\n return np.array(r)\n\n # only accepts 1 image at a time. Batch is used for gradient evaluations at different points\n def attack_batch(self, img, lab):\n \"\"\"\n Run the attack on a batch of images and labels.\n \"\"\"\n def compare(x,y):\n if not isinstance(x, (float, int, np.int64)):\n x = np.copy(x)\n if self.TARGETED:\n x[y] -= self.CONFIDENCE\n else:\n x[y] += self.CONFIDENCE\n x = np.argmax(x)\n if self.TARGETED:\n return x == y\n else:\n return x != y\n\n # remove the extra batch dimension\n if len(img.shape) == 4:\n img = img[0]\n if len(lab.shape) == 2:\n lab = lab[0]\n # convert to tanh-space\n if self.use_tanh:\n img = np.arctanh(img*1.999999)\n\n # set the lower and upper bounds accordingly\n lower_bound = 0.0\n CONST = self.initial_const\n upper_bound = 1e10\n\n # convert img to float32 to avoid numba error\n img = img.astype(np.float32)\n\n # set the upper and lower bounds for the modifier\n if not self.use_tanh:\n self.modifier_up = 0.5 - img.reshape(-1)\n self.modifier_down = -0.5 - img.reshape(-1)\n\n # clear the modifier\n if not self.load_checkpoint:\n if self.use_resize:\n self.resize_img(self.resize_init_size, self.resize_init_size, True)\n else:\n self.real_modifier.fill(0.0)\n\n # the best l2, score, and image attack\n o_best_const = CONST\n o_bestl2 = 1e10\n o_bestscore = -1\n o_bestattack = img\n \n for outer_step in range(self.BINARY_SEARCH_STEPS):\n print(o_bestl2)\n \n bestl2 = 1e10\n bestscore = -1\n\n # The last iteration (if we run many steps) repeat the search once.\n if self.repeat == True and outer_step == self.BINARY_SEARCH_STEPS-1:\n CONST = upper_bound\n\n # set the variables so that we don't have to send them over again\n self.sess.run(self.setup, {self.assign_timg: img,\n self.assign_tlab: lab,\n self.assign_const: CONST})\n\n # use the current best model\n # np.copyto(self.real_modifier, o_bestattack - img)\n # use the model left by last constant change\n \n prev = 1e6\n train_timer = 0.0\n last_loss1 = 1.0\n if not self.load_checkpoint:\n if self.use_resize:\n self.resize_img(self.resize_init_size, self.resize_init_size, True)\n else:\n self.real_modifier.fill(0.0)\n # reset ADAM status\n self.mt.fill(0.0)\n self.vt.fill(0.0)\n self.adam_epoch.fill(1)\n self.stage = 0\n multiplier = 1\n eval_costs = 0\n if self.solver_name != \"fake_zero\":\n multiplier = 24\n for iteration in range(self.start_iter, self.MAX_ITERATIONS):\n if self.use_resize:\n if iteration == 2000:\n # if iteration == 2000 // 24:\n self.resize_img(64,64)\n if iteration == 10000:\n # if iteration == 2000 // 24 + (10000 - 2000) // 96:\n self.resize_img(128,128)\n # if iteration == 200*30:\n # if iteration == 250 * multiplier:\n # self.resize_img(256,256)\n # print out the losses every 10%\n if iteration%(self.print_every) == 0:\n # print(iteration,self.sess.run((self.loss,self.real,self.other,self.loss1,self.loss2), feed_dict={self.modifier: self.real_modifier}))\n loss, real, other, loss1, loss2 = self.sess.run((self.loss,self.real,self.other,self.loss1,self.loss2), feed_dict={self.modifier: self.real_modifier})\n print(\"[STATS][L2] iter = {}, cost = {}, time = {:.3f}, size = {}, loss = {:.5g}, real = {:.5g}, other = {:.5g}, loss1 = {:.5g}, loss2 = {:.5g}\".format(iteration, eval_costs, train_timer, self.real_modifier.shape, loss[0], real[0], other[0], loss1[0], loss2[0]))\n sys.stdout.flush()\n # np.save('black_iter_{}'.format(iteration), self.real_modifier)\n\n attack_begin_time = time.time()\n # perform the attack \n if self.solver_name == \"fake_zero\":\n l, l2, loss1, loss2, score, nimg = self.fake_blackbox_optimizer()\n else:\n l, l2, loss1, loss2, score, nimg = self.blackbox_optimizer(iteration)\n # l = self.blackbox_optimizer(iteration)\n\n if self.solver_name == \"fake_zero\":\n eval_costs += np.prod(self.real_modifier.shape)\n else:\n eval_costs += self.batch_size\n\n # reset ADAM states when a valid example has been found\n if loss1 == 0.0 and last_loss1 != 0.0 and self.stage == 0:\n # we have reached the fine tunning point\n # reset ADAM to avoid overshoot\n if self.reset_adam_after_found:\n self.mt.fill(0.0)\n self.vt.fill(0.0)\n self.adam_epoch.fill(1)\n self.stage = 1\n last_loss1 = loss1\n\n # check if we should abort search if we're getting nowhere.\n # if self.ABORT_EARLY and iteration%(self.MAX_ITERATIONS//10) == 0:\n if self.ABORT_EARLY and iteration % self.early_stop_iters == 0:\n if l > prev*.9999:\n print(\"Early stopping because there is no improvement\")\n break\n prev = l\n\n # adjust the best result found so far\n # the best attack should have the target class with the largest value,\n # and has smallest l2 distance\n if l2 < bestl2 and compare(score, np.argmax(lab)):\n bestl2 = l2\n bestscore = np.argmax(score)\n if l2 < o_bestl2 and compare(score, np.argmax(lab)):\n # print a message if it is the first attack found\n if o_bestl2 == 1e10:\n print(\"[STATS][L3](First valid attack found!) iter = {}, cost = {}, time = {:.3f}, size = {}, loss = {:.5g}, loss1 = {:.5g}, loss2 = {:.5g}, l2 = {:.5g}\".format(iteration, eval_costs, train_timer, self.real_modifier.shape, l, loss1, loss2, l2))\n sys.stdout.flush()\n o_bestl2 = l2\n o_bestscore = np.argmax(score)\n o_bestattack = nimg\n o_best_const = CONST\n\n train_timer += time.time() - attack_begin_time\n\n # adjust the constant as needed\n if compare(bestscore, np.argmax(lab)) and bestscore != -1:\n # success, divide const by two\n print('old constant: ', CONST)\n upper_bound = min(upper_bound,CONST)\n if upper_bound < 1e9:\n CONST = (lower_bound + upper_bound)/2\n print('new constant: ', CONST)\n else:\n # failure, either multiply by 10 if no solution found yet\n # or do binary search with the known upper bound\n print('old constant: ', CONST)\n lower_bound = max(lower_bound,CONST)\n if upper_bound < 1e9:\n CONST = (lower_bound + upper_bound)/2\n else:\n CONST *= 10\n print('new constant: ', CONST)\n\n # return the best solution found\n return o_bestattack, o_best_const\n\n" }, { "path": "labels/imagenet_val_to_carlini.py", "content": "#!/usr/bin/env python3\n\nimport os\nimport sys\nimport glob\n\nf = open('label2num.txt')\n\nmapping = {}\nfor line in f:\n l = line.strip().split(':')\n mapping[l[1]] = l[0]\n\nprint(\"Total {} classes loaded\".format(len(mapping)))\n\nos.system(\"mkdir -p imgs\")\n\nfile_list = glob.glob('val/**/*.JPEG', recursive=True)\nprint(\"Total {} files found\".format(len(file_list)))\n\ncur = 1\ntotal = len(file_list)\n\nfor img_path in file_list:\n s = img_path.split('/')[1]\n n = os.path.splitext(os.path.basename(img_path))[0].split('_')[2]\n label = mapping[s]\n os.system(\"cp {} imgs/{}.{}.jpg\".format(img_path, label, n))\n if cur % 1000 == 0:\n print(\"{}/{} finished \".format(cur, total))\n cur += 1\n\nprint()\n\n" }, { "path": "labels/label2num.txt", "content": "1:n01440764:tench, Tinca tinca\n2:n01443537:goldfish, Carassius auratus\n3:n01484850:great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias\n4:n01491361:tiger shark, Galeocerdo cuvieri\n5:n01494475:hammerhead, hammerhead shark\n6:n01496331:electric ray, crampfish, numbfish, torpedo\n7:n01498041:stingray\n8:n01514668:cock\n9:n01514859:hen\n10:n01518878:ostrich, Struthio camelus\n11:n01530575:brambling, Fringilla montifringilla\n12:n01531178:goldfinch, Carduelis carduelis\n13:n01532829:house finch, linnet, Carpodacus mexicanus\n14:n01534433:junco, snowbird\n15:n01537544:indigo bunting, indigo finch, indigo bird, Passerina cyanea\n16:n01558993:robin, American robin, Turdus migratorius\n17:n01560419:bulbul\n18:n01580077:jay\n19:n01582220:magpie\n20:n01592084:chickadee\n21:n01601694:water ouzel, dipper\n22:n01608432:kite\n23:n01614925:bald eagle, American eagle, Haliaeetus leucocephalus\n24:n01616318:vulture\n25:n01622779:great grey owl, great gray owl, Strix nebulosa\n26:n01629819:European fire salamander, Salamandra salamandra\n27:n01630670:common newt, Triturus vulgaris\n28:n01631663:eft\n29:n01632458:spotted salamander, Ambystoma maculatum\n30:n01632777:axolotl, mud puppy, Ambystoma mexicanum\n31:n01641577:bullfrog, Rana catesbeiana\n32:n01644373:tree frog, tree-frog\n33:n01644900:tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui\n34:n01664065:loggerhead, loggerhead turtle, Caretta caretta\n35:n01665541:leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea\n36:n01667114:mud turtle\n37:n01667778:terrapin\n38:n01669191:box turtle, box tortoise\n39:n01675722:banded gecko\n40:n01677366:common iguana, iguana, Iguana iguana\n41:n01682714:American chameleon, anole, Anolis carolinensis\n42:n01685808:whiptail, whiptail lizard\n43:n01687978:agama\n44:n01688243:frilled lizard, Chlamydosaurus kingi\n45:n01689811:alligator lizard\n46:n01692333:Gila monster, Heloderma suspectum\n47:n01693334:green lizard, Lacerta viridis\n48:n01694178:African chameleon, Chamaeleo chamaeleon\n49:n01695060:Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis\n50:n01697457:African crocodile, Nile crocodile, Crocodylus niloticus\n51:n01698640:American alligator, Alligator mississipiensis\n52:n01704323:triceratops\n53:n01728572:thunder snake, worm snake, Carphophis amoenus\n54:n01728920:ringneck snake, ring-necked snake, ring snake\n55:n01729322:hognose snake, puff adder, sand viper\n56:n01729977:green snake, grass snake\n57:n01734418:king snake, kingsnake\n58:n01735189:garter snake, grass snake\n59:n01737021:water snake\n60:n01739381:vine snake\n61:n01740131:night snake, Hypsiglena torquata\n62:n01742172:boa constrictor, Constrictor constrictor\n63:n01744401:rock python, rock snake, Python sebae\n64:n01748264:Indian cobra, Naja naja\n65:n01749939:green mamba\n66:n01751748:sea snake\n67:n01753488:horned viper, cerastes, sand viper, horned asp, Cerastes cornutus\n68:n01755581:diamondback, diamondback rattlesnake, Crotalus adamanteus\n69:n01756291:sidewinder, horned rattlesnake, Crotalus cerastes\n70:n01768244:trilobite\n71:n01770081:harvestman, daddy longlegs, Phalangium opilio\n72:n01770393:scorpion\n73:n01773157:black and gold garden spider, Argiope aurantia\n74:n01773549:barn spider, Araneus cavaticus\n75:n01773797:garden spider, Aranea diademata\n76:n01774384:black widow, Latrodectus mactans\n77:n01774750:tarantula\n78:n01775062:wolf spider, hunting spider\n79:n01776313:tick\n80:n01784675:centipede\n81:n01795545:black grouse\n82:n01796340:ptarmigan\n83:n01797886:ruffed grouse, partridge, Bonasa umbellus\n84:n01798484:prairie chicken, prairie grouse, prairie fowl\n85:n01806143:peacock\n86:n01806567:quail\n87:n01807496:partridge\n88:n01817953:African grey, African gray, Psittacus erithacus\n89:n01818515:macaw\n90:n01819313:sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita\n91:n01820546:lorikeet\n92:n01824575:coucal\n93:n01828970:bee eater\n94:n01829413:hornbill\n95:n01833805:hummingbird\n96:n01843065:jacamar\n97:n01843383:toucan\n98:n01847000:drake\n99:n01855032:red-breasted merganser, Mergus serrator\n100:n01855672:goose\n101:n01860187:black swan, Cygnus atratus\n102:n01871265:tusker\n103:n01872401:echidna, spiny anteater, anteater\n104:n01873310:platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus\n105:n01877812:wallaby, brush kangaroo\n106:n01882714:koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus\n107:n01883070:wombat\n108:n01910747:jellyfish\n109:n01914609:sea anemone, anemone\n110:n01917289:brain coral\n111:n01924916:flatworm, platyhelminth\n112:n01930112:nematode, nematode worm, roundworm\n113:n01943899:conch\n114:n01944390:snail\n115:n01945685:slug\n116:n01950731:sea slug, nudibranch\n117:n01955084:chiton, coat-of-mail shell, sea cradle, polyplacophore\n118:n01968897:chambered nautilus, pearly nautilus, nautilus\n119:n01978287:Dungeness crab, Cancer magister\n120:n01978455:rock crab, Cancer irroratus\n121:n01980166:fiddler crab\n122:n01981276:king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica\n123:n01983481:American lobster, Northern lobster, Maine lobster, Homarus americanus\n124:n01984695:spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish\n125:n01985128:crayfish, crawfish, crawdad, crawdaddy\n126:n01986214:hermit crab\n127:n01990800:isopod\n128:n02002556:white stork, Ciconia ciconia\n129:n02002724:black stork, Ciconia nigra\n130:n02006656:spoonbill\n131:n02007558:flamingo\n132:n02009229:little blue heron, Egretta caerulea\n133:n02009912:American egret, great white heron, Egretta albus\n134:n02011460:bittern\n518:n02012849:crane\n136:n02013706:limpkin, Aramus pictus\n137:n02017213:European gallinule, Porphyrio porphyrio\n138:n02018207:American coot, marsh hen, mud hen, water hen, Fulica americana\n139:n02018795:bustard\n140:n02025239:ruddy turnstone, Arenaria interpres\n141:n02027492:red-backed sandpiper, dunlin, Erolia alpina\n142:n02028035:redshank, Tringa totanus\n143:n02033041:dowitcher\n144:n02037110:oystercatcher, oyster catcher\n145:n02051845:pelican\n146:n02056570:king penguin, Aptenodytes patagonica\n147:n02058221:albatross, mollymawk\n148:n02066245:grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus\n149:n02071294:killer whale, killer, orca, grampus, sea wolf, Orcinus orca\n150:n02074367:dugong, Dugong dugon\n151:n02077923:sea lion\n152:n02085620:Chihuahua\n153:n02085782:Japanese spaniel\n154:n02085936:Maltese dog, Maltese terrier, Maltese\n155:n02086079:Pekinese, Pekingese, Peke\n156:n02086240:Shih-Tzu\n157:n02086646:Blenheim spaniel\n158:n02086910:papillon\n159:n02087046:toy terrier\n160:n02087394:Rhodesian ridgeback\n161:n02088094:Afghan hound, Afghan\n162:n02088238:basset, basset hound\n163:n02088364:beagle\n164:n02088466:bloodhound, sleuthhound\n165:n02088632:bluetick\n166:n02089078:black-and-tan coonhound\n167:n02089867:Walker hound, Walker foxhound\n168:n02089973:English foxhound\n169:n02090379:redbone\n170:n02090622:borzoi, Russian wolfhound\n171:n02090721:Irish wolfhound\n172:n02091032:Italian greyhound\n173:n02091134:whippet\n174:n02091244:Ibizan hound, Ibizan Podenco\n175:n02091467:Norwegian elkhound, elkhound\n176:n02091635:otterhound, otter hound\n177:n02091831:Saluki, gazelle hound\n178:n02092002:Scottish deerhound, deerhound\n179:n02092339:Weimaraner\n180:n02093256:Staffordshire bullterrier, Staffordshire bull terrier\n181:n02093428:American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier\n182:n02093647:Bedlington terrier\n183:n02093754:Border terrier\n184:n02093859:Kerry blue terrier\n185:n02093991:Irish terrier\n186:n02094114:Norfolk terrier\n187:n02094258:Norwich terrier\n188:n02094433:Yorkshire terrier\n189:n02095314:wire-haired fox terrier\n190:n02095570:Lakeland terrier\n191:n02095889:Sealyham terrier, Sealyham\n192:n02096051:Airedale, Airedale terrier\n193:n02096177:cairn, cairn terrier\n194:n02096294:Australian terrier\n195:n02096437:Dandie Dinmont, Dandie Dinmont terrier\n196:n02096585:Boston bull, Boston terrier\n197:n02097047:miniature schnauzer\n198:n02097130:giant schnauzer\n199:n02097209:standard schnauzer\n200:n02097298:Scotch terrier, Scottish terrier, Scottie\n201:n02097474:Tibetan terrier, chrysanthemum dog\n202:n02097658:silky terrier, Sydney silky\n203:n02098105:soft-coated wheaten terrier\n204:n02098286:West Highland white terrier\n205:n02098413:Lhasa, Lhasa apso\n206:n02099267:flat-coated retriever\n207:n02099429:curly-coated retriever\n208:n02099601:golden retriever\n209:n02099712:Labrador retriever\n210:n02099849:Chesapeake Bay retriever\n211:n02100236:German short-haired pointer\n212:n02100583:vizsla, Hungarian pointer\n213:n02100735:English setter\n214:n02100877:Irish setter, red setter\n215:n02101006:Gordon setter\n216:n02101388:Brittany spaniel\n217:n02101556:clumber, clumber spaniel\n218:n02102040:English springer, English springer spaniel\n219:n02102177:Welsh springer spaniel\n220:n02102318:cocker spaniel, English cocker spaniel, cocker\n221:n02102480:Sussex spaniel\n222:n02102973:Irish water spaniel\n223:n02104029:kuvasz\n224:n02104365:schipperke\n225:n02105056:groenendael\n226:n02105162:malinois\n227:n02105251:briard\n228:n02105412:kelpie\n229:n02105505:komondor\n230:n02105641:Old English sheepdog, bobtail\n231:n02105855:Shetland sheepdog, Shetland sheep dog, Shetland\n232:n02106030:collie\n233:n02106166:Border collie\n234:n02106382:Bouvier des Flandres, Bouviers des Flandres\n235:n02106550:Rottweiler\n236:n02106662:German shepherd, German shepherd dog, German police dog, alsatian\n237:n02107142:Doberman, Doberman pinscher\n238:n02107312:miniature pinscher\n239:n02107574:Greater Swiss Mountain dog\n240:n02107683:Bernese mountain dog\n241:n02107908:Appenzeller\n242:n02108000:EntleBucher\n243:n02108089:boxer\n244:n02108422:bull mastiff\n245:n02108551:Tibetan mastiff\n246:n02108915:French bulldog\n247:n02109047:Great Dane\n248:n02109525:Saint Bernard, St Bernard\n249:n02109961:Eskimo dog, husky\n250:n02110063:malamute, malemute, Alaskan malamute\n251:n02110185:Siberian husky\n252:n02110341:dalmatian, coach dog, carriage dog\n253:n02110627:affenpinscher, monkey pinscher, monkey dog\n254:n02110806:basenji\n255:n02110958:pug, pug-dog\n256:n02111129:Leonberg\n257:n02111277:Newfoundland, Newfoundland dog\n258:n02111500:Great Pyrenees\n259:n02111889:Samoyed, Samoyede\n260:n02112018:Pomeranian\n261:n02112137:chow, chow chow\n262:n02112350:keeshond\n263:n02112706:Brabancon griffon\n264:n02113023:Pembroke, Pembroke Welsh corgi\n265:n02113186:Cardigan, Cardigan Welsh corgi\n266:n02113624:toy poodle\n267:n02113712:miniature poodle\n268:n02113799:standard poodle\n269:n02113978:Mexican hairless\n270:n02114367:timber wolf, grey wolf, gray wolf, Canis lupus\n271:n02114548:white wolf, Arctic wolf, Canis lupus tundrarum\n272:n02114712:red wolf, maned wolf, Canis rufus, Canis niger\n273:n02114855:coyote, prairie wolf, brush wolf, Canis latrans\n274:n02115641:dingo, warrigal, warragal, Canis dingo\n275:n02115913:dhole, Cuon alpinus\n276:n02116738:African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus\n277:n02117135:hyena, hyaena\n278:n02119022:red fox, Vulpes vulpes\n279:n02119789:kit fox, Vulpes macrotis\n280:n02120079:Arctic fox, white fox, Alopex lagopus\n281:n02120505:grey fox, gray fox, Urocyon cinereoargenteus\n282:n02123045:tabby, tabby cat\n283:n02123159:tiger cat\n284:n02123394:Persian cat\n285:n02123597:Siamese cat, Siamese\n286:n02124075:Egyptian cat\n287:n02125311:cougar, puma, catamount, mountain lion, painter, panther, Felis concolor\n288:n02127052:lynx, catamount\n289:n02128385:leopard, Panthera pardus\n290:n02128757:snow leopard, ounce, Panthera uncia\n291:n02128925:jaguar, panther, Panthera onca, Felis onca\n292:n02129165:lion, king of beasts, Panthera leo\n293:n02129604:tiger, Panthera tigris\n294:n02130308:cheetah, chetah, Acinonyx jubatus\n295:n02132136:brown bear, bruin, Ursus arctos\n296:n02133161:American black bear, black bear, Ursus americanus, Euarctos americanus\n297:n02134084:ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus\n298:n02134418:sloth bear, Melursus ursinus, Ursus ursinus\n299:n02137549:mongoose\n300:n02138441:meerkat, mierkat\n301:n02165105:tiger beetle\n302:n02165456:ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle\n303:n02167151:ground beetle, carabid beetle\n304:n02168699:long-horned beetle, longicorn, longicorn beetle\n305:n02169497:leaf beetle, chrysomelid\n306:n02172182:dung beetle\n307:n02174001:rhinoceros beetle\n308:n02177972:weevil\n309:n02190166:fly\n310:n02206856:bee\n311:n02219486:ant, emmet, pismire\n312:n02226429:grasshopper, hopper\n313:n02229544:cricket\n314:n02231487:walking stick, walkingstick, stick insect\n315:n02233338:cockroach, roach\n316:n02236044:mantis, mantid\n317:n02256656:cicada, cicala\n318:n02259212:leafhopper\n319:n02264363:lacewing, lacewing fly\n320:n02268443:dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk\n321:n02268853:damselfly\n322:n02276258:admiral\n323:n02277742:ringlet, ringlet butterfly\n324:n02279972:monarch, monarch butterfly, milkweed butterfly, Danaus plexippus\n325:n02280649:cabbage butterfly\n326:n02281406:sulphur butterfly, sulfur butterfly\n327:n02281787:lycaenid, lycaenid butterfly\n328:n02317335:starfish, sea star\n329:n02319095:sea urchin\n330:n02321529:sea cucumber, holothurian\n331:n02325366:wood rabbit, cottontail, cottontail rabbit\n332:n02326432:hare\n333:n02328150:Angora, Angora rabbit\n334:n02342885:hamster\n335:n02346627:porcupine, hedgehog\n336:n02356798:fox squirrel, eastern fox squirrel, Sciurus niger\n337:n02361337:marmot\n338:n02363005:beaver\n339:n02364673:guinea pig, Cavia cobaya\n340:n02389026:sorrel\n341:n02391049:zebra\n342:n02395406:hog, pig, grunter, squealer, Sus scrofa\n343:n02396427:wild boar, boar, Sus scrofa\n344:n02397096:warthog\n345:n02398521:hippopotamus, hippo, river horse, Hippopotamus amphibius\n346:n02403003:ox\n347:n02408429:water buffalo, water ox, Asiatic buffalo, Bubalus bubalis\n348:n02410509:bison\n349:n02412080:ram, tup\n350:n02415577:bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis\n351:n02417914:ibex, Capra ibex\n352:n02422106:hartebeest\n353:n02422699:impala, Aepyceros melampus\n354:n02423022:gazelle\n355:n02437312:Arabian camel, dromedary, Camelus dromedarius\n356:n02437616:llama\n357:n02441942:weasel\n358:n02442845:mink\n359:n02443114:polecat, fitch, foulmart, foumart, Mustela putorius\n360:n02443484:black-footed ferret, ferret, Mustela nigripes\n361:n02444819:otter\n362:n02445715:skunk, polecat, wood pussy\n363:n02447366:badger\n364:n02454379:armadillo\n365:n02457408:three-toed sloth, ai, Bradypus tridactylus\n366:n02480495:orangutan, orang, orangutang, Pongo pygmaeus\n367:n02480855:gorilla, Gorilla gorilla\n368:n02481823:chimpanzee, chimp, Pan troglodytes\n369:n02483362:gibbon, Hylobates lar\n370:n02483708:siamang, Hylobates syndactylus, Symphalangus syndactylus\n371:n02484975:guenon, guenon monkey\n372:n02486261:patas, hussar monkey, Erythrocebus patas\n373:n02486410:baboon\n374:n02487347:macaque\n375:n02488291:langur\n376:n02488702:colobus, colobus monkey\n377:n02489166:proboscis monkey, Nasalis larvatus\n378:n02490219:marmoset\n379:n02492035:capuchin, ringtail, Cebus capucinus\n380:n02492660:howler monkey, howler\n381:n02493509:titi, titi monkey\n382:n02493793:spider monkey, Ateles geoffroyi\n383:n02494079:squirrel monkey, Saimiri sciureus\n384:n02497673:Madagascar cat, ring-tailed lemur, Lemur catta\n385:n02500267:indri, indris, Indri indri, Indri brevicaudatus\n386:n02504013:Indian elephant, Elephas maximus\n387:n02504458:African elephant, Loxodonta africana\n388:n02509815:lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens\n389:n02510455:giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca\n390:n02514041:barracouta, snoek\n391:n02526121:eel\n392:n02536864:coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch\n393:n02606052:rock beauty, Holocanthus tricolor\n394:n02607072:anemone fish\n395:n02640242:sturgeon\n396:n02641379:gar, garfish, garpike, billfish, Lepisosteus osseus\n397:n02643566:lionfish\n398:n02655020:puffer, pufferfish, blowfish, globefish\n399:n02666196:abacus\n400:n02667093:abaya\n401:n02669723:academic gown, academic robe, judge's robe\n402:n02672831:accordion, piano accordion, squeeze box\n403:n02676566:acoustic guitar\n404:n02687172:aircraft carrier, carrier, flattop, attack aircraft carrier\n405:n02690373:airliner\n406:n02692877:airship, dirigible\n407:n02699494:altar\n408:n02701002:ambulance\n409:n02704792:amphibian, amphibious vehicle\n410:n02708093:analog clock\n411:n02727426:apiary, bee house\n412:n02730930:apron\n413:n02747177:ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin\n414:n02749479:assault rifle, assault gun\n415:n02769748:backpack, back pack, knapsack, packsack, rucksack, haversack\n416:n02776631:bakery, bakeshop, bakehouse\n417:n02777292:balance beam, beam\n418:n02782093:balloon\n419:n02783161:ballpoint, ballpoint pen, ballpen, Biro\n420:n02786058:Band Aid\n421:n02787622:banjo\n422:n02788148:bannister, banister, balustrade, balusters, handrail\n423:n02790996:barbell\n424:n02791124:barber chair\n425:n02791270:barbershop\n426:n02793495:barn\n427:n02794156:barometer\n428:n02795169:barrel, cask\n429:n02797295:barrow, garden cart, lawn cart, wheelbarrow\n430:n02799071:baseball\n431:n02802426:basketball\n432:n02804414:bassinet\n433:n02804610:bassoon\n434:n02807133:bathing cap, swimming cap\n435:n02808304:bath towel\n436:n02808440:bathtub, bathing tub, bath, tub\n437:n02814533:beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon\n438:n02814860:beacon, lighthouse, beacon light, pharos\n439:n02815834:beaker\n440:n02817516:bearskin, busby, shako\n441:n02823428:beer bottle\n442:n02823750:beer glass\n443:n02825657:bell cote, bell cot\n444:n02834397:bib\n445:n02835271:bicycle-built-for-two, tandem bicycle, tandem\n446:n02837789:bikini, two-piece\n447:n02840245:binder, ring-binder\n448:n02841315:binoculars, field glasses, opera glasses\n449:n02843684:birdhouse\n450:n02859443:boathouse\n451:n02860847:bobsled, bobsleigh, bob\n452:n02865351:bolo tie, bolo, bola tie, bola\n453:n02869837:bonnet, poke bonnet\n454:n02870880:bookcase\n455:n02871525:bookshop, bookstore, bookstall\n456:n02877765:bottlecap\n457:n02879718:bow\n458:n02883205:bow tie, bow-tie, bowtie\n459:n02892201:brass, memorial tablet, plaque\n460:n02892767:brassiere, bra, bandeau\n461:n02894605:breakwater, groin, groyne, mole, bulwark, seawall, jetty\n462:n02895154:breastplate, aegis, egis\n463:n02906734:broom\n464:n02909870:bucket, pail\n465:n02910353:buckle\n466:n02916936:bulletproof vest\n467:n02917067:bullet train, bullet\n468:n02927161:butcher shop, meat market\n469:n02930766:cab, hack, taxi, taxicab\n470:n02939185:caldron, cauldron\n471:n02948072:candle, taper, wax light\n472:n02950826:cannon\n473:n02951358:canoe\n474:n02951585:can opener, tin opener\n475:n02963159:cardigan\n476:n02965783:car mirror\n477:n02966193:carousel, carrousel, merry-go-round, roundabout, whirligig\n478:n02966687:carpenter's kit, tool kit\n479:n02971356:carton\n480:n02974003:car wheel\n481:n02977058:cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM\n482:n02978881:cassette\n483:n02979186:cassette player\n484:n02980441:castle\n485:n02981792:catamaran\n486:n02988304:CD player\n487:n02992211:cello, violoncello\n488:n02992529:cellular telephone, cellular phone, cellphone, cell, mobile phone\n489:n02999410:chain\n490:n03000134:chainlink fence\n491:n03000247:chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour\n492:n03000684:chain saw, chainsaw\n493:n03014705:chest\n494:n03016953:chiffonier, commode\n495:n03017168:chime, bell, gong\n496:n03018349:china cabinet, china closet\n497:n03026506:Christmas stocking\n498:n03028079:church, church building\n499:n03032252:cinema, movie theater, movie theatre, movie house, picture palace\n500:n03041632:cleaver, meat cleaver, chopper\n501:n03042490:cliff dwelling\n502:n03045698:cloak\n503:n03047690:clog, geta, patten, sabot\n504:n03062245:cocktail shaker\n505:n03063599:coffee mug\n506:n03063689:coffeepot\n507:n03065424:coil, spiral, volute, whorl, helix\n508:n03075370:combination lock\n509:n03085013:computer keyboard, keypad\n510:n03089624:confectionery, confectionary, candy store\n511:n03095699:container ship, containership, container vessel\n512:n03100240:convertible\n513:n03109150:corkscrew, bottle screw\n514:n03110669:cornet, horn, trumpet, trump\n515:n03124043:cowboy boot\n516:n03124170:cowboy hat, ten-gallon hat\n517:n03125729:cradle\n518:n03126707:crane\n519:n03127747:crash helmet\n520:n03127925:crate\n521:n03131574:crib, cot\n522:n03133878:Crock Pot\n523:n03134739:croquet ball\n524:n03141823:crutch\n525:n03146219:cuirass\n526:n03160309:dam, dike, dyke\n527:n03179701:desk\n528:n03180011:desktop computer\n529:n03187595:dial telephone, dial phone\n530:n03188531:diaper, nappy, napkin\n531:n03196217:digital clock\n532:n03197337:digital watch\n533:n03201208:dining table, board\n534:n03207743:dishrag, dishcloth\n535:n03207941:dishwasher, dish washer, dishwashing machine\n536:n03208938:disk brake, disc brake\n537:n03216828:dock, dockage, docking facility\n538:n03218198:dogsled, dog sled, dog sleigh\n539:n03220513:dome\n540:n03223299:doormat, welcome mat\n541:n03240683:drilling platform, offshore rig\n542:n03249569:drum, membranophone, tympan\n543:n03250847:drumstick\n544:n03255030:dumbbell\n545:n03259280:Dutch oven\n546:n03271574:electric fan, blower\n547:n03272010:electric guitar\n548:n03272562:electric locomotive\n549:n03290653:entertainment center\n550:n03291819:envelope\n551:n03297495:espresso maker\n552:n03314780:face powder\n553:n03325584:feather boa, boa\n554:n03337140:file, file cabinet, filing cabinet\n555:n03344393:fireboat\n556:n03345487:fire engine, fire truck\n557:n03347037:fire screen, fireguard\n558:n03355925:flagpole, flagstaff\n559:n03372029:flute, transverse flute\n560:n03376595:folding chair\n561:n03379051:football helmet\n562:n03384352:forklift\n563:n03388043:fountain\n564:n03388183:fountain pen\n565:n03388549:four-poster\n566:n03393912:freight car\n567:n03394916:French horn, horn\n568:n03400231:frying pan, frypan, skillet\n569:n03404251:fur coat\n570:n03417042:garbage truck, dustcart\n571:n03424325:gasmask, respirator, gas helmet\n572:n03425413:gas pump, gasoline pump, petrol pump, island dispenser\n573:n03443371:goblet\n574:n03444034:go-kart\n575:n03445777:golf ball\n576:n03445924:golfcart, golf cart\n577:n03447447:gondola\n578:n03447721:gong, tam-tam\n579:n03450230:gown\n580:n03452741:grand piano, grand\n581:n03457902:greenhouse, nursery, glasshouse\n582:n03459775:grille, radiator grille\n583:n03461385:grocery store, grocery, food market, market\n584:n03467068:guillotine\n585:n03476684:hair slide\n586:n03476991:hair spray\n587:n03478589:half track\n588:n03481172:hammer\n589:n03482405:hamper\n590:n03483316:hand blower, blow dryer, blow drier, hair dryer, hair drier\n591:n03485407:hand-held computer, hand-held microcomputer\n592:n03485794:handkerchief, hankie, hanky, hankey\n593:n03492542:hard disc, hard disk, fixed disk\n594:n03494278:harmonica, mouth organ, harp, mouth harp\n595:n03495258:harp\n596:n03496892:harvester, reaper\n597:n03498962:hatchet\n598:n03527444:holster\n599:n03529860:home theater, home theatre\n600:n03530642:honeycomb\n601:n03532672:hook, claw\n602:n03534580:hoopskirt, crinoline\n603:n03535780:horizontal bar, high bar\n604:n03538406:horse cart, horse-cart\n605:n03544143:hourglass\n606:n03584254:iPod\n607:n03584829:iron, smoothing iron\n608:n03590841:jack-o'-lantern\n609:n03594734:jean, blue jean, denim\n610:n03594945:jeep, landrover\n611:n03595614:jersey, T-shirt, tee shirt\n612:n03598930:jigsaw puzzle\n613:n03599486:jinrikisha, ricksha, rickshaw\n614:n03602883:joystick\n615:n03617480:kimono\n616:n03623198:knee pad\n617:n03627232:knot\n618:n03630383:lab coat, laboratory coat\n619:n03633091:ladle\n620:n03637318:lampshade, lamp shade\n621:n03642806:laptop, laptop computer\n622:n03649909:lawn mower, mower\n623:n03657121:lens cap, lens cover\n624:n03658185:letter opener, paper knife, paperknife\n625:n03661043:library\n626:n03662601:lifeboat\n627:n03666591:lighter, light, igniter, ignitor\n628:n03670208:limousine, limo\n629:n03673027:liner, ocean liner\n630:n03676483:lipstick, lip rouge\n631:n03680355:Loafer\n632:n03690938:lotion\n633:n03691459:loudspeaker, speaker, speaker unit, loudspeaker system, speaker system\n634:n03692522:loupe, jeweler's loupe\n635:n03697007:lumbermill, sawmill\n636:n03706229:magnetic compass\n637:n03709823:mailbag, postbag\n638:n03710193:mailbox, letter box\n639:n03710637:maillot\n640:n03710721:maillot, tank suit\n641:n03717622:manhole cover\n642:n03720891:maraca\n643:n03721384:marimba, xylophone\n644:n03724870:mask\n645:n03729826:matchstick\n646:n03733131:maypole\n647:n03733281:maze, labyrinth\n648:n03733805:measuring cup\n649:n03742115:medicine chest, medicine cabinet\n650:n03743016:megalith, megalithic structure\n651:n03759954:microphone, mike\n652:n03761084:microwave, microwave oven\n653:n03763968:military uniform\n654:n03764736:milk can\n655:n03769881:minibus\n656:n03770439:miniskirt, mini\n657:n03770679:minivan\n658:n03773504:missile\n659:n03775071:mitten\n660:n03775546:mixing bowl\n661:n03776460:mobile home, manufactured home\n662:n03777568:Model T\n663:n03777754:modem\n664:n03781244:monastery\n665:n03782006:monitor\n666:n03785016:moped\n667:n03786901:mortar\n668:n03787032:mortarboard\n669:n03788195:mosque\n670:n03788365:mosquito net\n671:n03791053:motor scooter, scooter\n672:n03792782:mountain bike, all-terrain bike, off-roader\n673:n03792972:mountain tent\n674:n03793489:mouse, computer mouse\n675:n03794056:mousetrap\n676:n03796401:moving van\n677:n03803284:muzzle\n678:n03804744:nail\n679:n03814639:neck brace\n680:n03814906:necklace\n681:n03825788:nipple\n682:n03832673:notebook, notebook computer\n683:n03837869:obelisk\n684:n03838899:oboe, hautboy, hautbois\n685:n03840681:ocarina, sweet potato\n686:n03841143:odometer, hodometer, mileometer, milometer\n687:n03843555:oil filter\n688:n03854065:organ, pipe organ\n689:n03857828:oscilloscope, scope, cathode-ray oscilloscope, CRO\n690:n03866082:overskirt\n691:n03868242:oxcart\n692:n03868863:oxygen mask\n693:n03871628:packet\n694:n03873416:paddle, boat paddle\n695:n03874293:paddlewheel, paddle wheel\n696:n03874599:padlock\n697:n03876231:paintbrush\n698:n03877472:pajama, pyjama, pj's, jammies\n699:n03877845:palace\n700:n03884397:panpipe, pandean pipe, syrinx\n701:n03887697:paper towel\n702:n03888257:parachute, chute\n703:n03888605:parallel bars, bars\n704:n03891251:park bench\n705:n03891332:parking meter\n706:n03895866:passenger car, coach, carriage\n707:n03899768:patio, terrace\n708:n03902125:pay-phone, pay-station\n709:n03903868:pedestal, plinth, footstall\n710:n03908618:pencil box, pencil case\n711:n03908714:pencil sharpener\n712:n03916031:perfume, essence\n713:n03920288:Petri dish\n714:n03924679:photocopier\n715:n03929660:pick, plectrum, plectron\n716:n03929855:pickelhaube\n717:n03930313:picket fence, paling\n718:n03930630:pickup, pickup truck\n719:n03933933:pier\n720:n03935335:piggy bank, penny bank\n721:n03937543:pill bottle\n722:n03938244:pillow\n723:n03942813:ping-pong ball\n724:n03944341:pinwheel\n725:n03947888:pirate, pirate ship\n726:n03950228:pitcher, ewer\n727:n03954731:plane, carpenter's plane, woodworking plane\n728:n03956157:planetarium\n729:n03958227:plastic bag\n730:n03961711:plate rack\n731:n03967562:plow, plough\n732:n03970156:plunger, plumber's helper\n733:n03976467:Polaroid camera, Polaroid Land camera\n734:n03976657:pole\n735:n03977966:police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria\n736:n03980874:poncho\n737:n03982430:pool table, billiard table, snooker table\n738:n03983396:pop bottle, soda bottle\n739:n03991062:pot, flowerpot\n740:n03992509:potter's wheel\n741:n03995372:power drill\n742:n03998194:prayer rug, prayer mat\n743:n04004767:printer\n744:n04005630:prison, prison house\n745:n04008634:projectile, missile\n746:n04009552:projector\n747:n04019541:puck, hockey puck\n748:n04023962:punching bag, punch bag, punching ball, punchball\n749:n04026417:purse\n750:n04033901:quill, quill pen\n751:n04033995:quilt, comforter, comfort, puff\n752:n04037443:racer, race car, racing car\n753:n04039381:racket, racquet\n754:n04040759:radiator\n755:n04041544:radio, wireless\n756:n04044716:radio telescope, radio reflector\n757:n04049303:rain barrel\n758:n04065272:recreational vehicle, RV, R.V.\n759:n04067472:reel\n760:n04069434:reflex camera\n761:n04070727:refrigerator, icebox\n762:n04074963:remote control, remote\n763:n04081281:restaurant, eating house, eating place, eatery\n764:n04086273:revolver, six-gun, six-shooter\n765:n04090263:rifle\n766:n04099969:rocking chair, rocker\n767:n04111531:rotisserie\n768:n04116512:rubber eraser, rubber, pencil eraser\n769:n04118538:rugby ball\n770:n04118776:rule, ruler\n771:n04120489:running shoe\n772:n04125021:safe\n773:n04127249:safety pin\n774:n04131690:saltshaker, salt shaker\n775:n04133789:sandal\n776:n04136333:sarong\n777:n04141076:sax, saxophone\n778:n04141327:scabbard\n779:n04141975:scale, weighing machine\n780:n04146614:school bus\n781:n04147183:schooner\n782:n04149813:scoreboard\n783:n04152593:screen, CRT screen\n784:n04153751:screw\n785:n04154565:screwdriver\n786:n04162706:seat belt, seatbelt\n787:n04179913:sewing machine\n788:n04192698:shield, buckler\n789:n04200800:shoe shop, shoe-shop, shoe store\n790:n04201297:shoji\n791:n04204238:shopping basket\n792:n04204347:shopping cart\n793:n04208210:shovel\n794:n04209133:shower cap\n795:n04209239:shower curtain\n796:n04228054:ski\n797:n04229816:ski mask\n798:n04235860:sleeping bag\n799:n04238763:slide rule, slipstick\n800:n04239074:sliding door\n801:n04243546:slot, one-armed bandit\n802:n04251144:snorkel\n803:n04252077:snowmobile\n804:n04252225:snowplow, snowplough\n805:n04254120:soap dispenser\n806:n04254680:soccer ball\n807:n04254777:sock\n808:n04258138:solar dish, solar collector, solar furnace\n809:n04259630:sombrero\n810:n04263257:soup bowl\n811:n04264628:space bar\n812:n04265275:space heater\n813:n04266014:space shuttle\n814:n04270147:spatula\n815:n04273569:speedboat\n816:n04275548:spider web, spider's web\n817:n04277352:spindle\n818:n04285008:sports car, sport car\n819:n04286575:spotlight, spot\n820:n04296562:stage\n821:n04310018:steam locomotive\n822:n04311004:steel arch bridge\n823:n04311174:steel drum\n824:n04317175:stethoscope\n825:n04325704:stole\n826:n04326547:stone wall\n827:n04328186:stopwatch, stop watch\n828:n04330267:stove\n829:n04332243:strainer\n830:n04335435:streetcar, tram, tramcar, trolley, trolley car\n831:n04336792:stretcher\n832:n04344873:studio couch, day bed\n833:n04346328:stupa, tope\n834:n04347754:submarine, pigboat, sub, U-boat\n835:n04350905:suit, suit of clothes\n836:n04355338:sundial\n837:n04355933:sunglass\n838:n04356056:sunglasses, dark glasses, shades\n839:n04357314:sunscreen, sunblock, sun blocker\n840:n04366367:suspension bridge\n841:n04367480:swab, swob, mop\n842:n04370456:sweatshirt\n843:n04371430:swimming trunks, bathing trunks\n844:n04371774:swing\n845:n04372370:switch, electric switch, electrical switch\n846:n04376876:syringe\n847:n04380533:table lamp\n848:n04389033:tank, army tank, armored combat vehicle, armoured combat vehicle\n849:n04392985:tape player\n850:n04398044:teapot\n851:n04399382:teddy, teddy bear\n852:n04404412:television, television system\n853:n04409515:tennis ball\n854:n04417672:thatch, thatched roof\n855:n04418357:theater curtain, theatre curtain\n856:n04423845:thimble\n857:n04428191:thresher, thrasher, threshing machine\n858:n04429376:throne\n859:n04435653:tile roof\n860:n04442312:toaster\n861:n04443257:tobacco shop, tobacconist shop, tobacconist\n862:n04447861:toilet seat\n863:n04456115:torch\n864:n04458633:totem pole\n865:n04461696:tow truck, tow car, wrecker\n866:n04462240:toyshop\n867:n04465501:tractor\n868:n04467665:trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi\n869:n04476259:tray\n870:n04479046:trench coat\n871:n04482393:tricycle, trike, velocipede\n872:n04483307:trimaran\n873:n04485082:tripod\n874:n04486054:triumphal arch\n875:n04487081:trolleybus, trolley coach, trackless trolley\n876:n04487394:trombone\n877:n04493381:tub, vat\n878:n04501370:turnstile\n879:n04505470:typewriter keyboard\n880:n04507155:umbrella\n881:n04509417:unicycle, monocycle\n882:n04515003:upright, upright piano\n883:n04517823:vacuum, vacuum cleaner\n884:n04522168:vase\n885:n04523525:vault\n886:n04525038:velvet\n887:n04525305:vending machine\n888:n04532106:vestment\n889:n04532670:viaduct\n890:n04536866:violin, fiddle\n891:n04540053:volleyball\n892:n04542943:waffle iron\n893:n04548280:wall clock\n894:n04548362:wallet, billfold, notecase, pocketbook\n895:n04550184:wardrobe, closet, press\n896:n04552348:warplane, military plane\n897:n04553703:washbasin, handbasin, washbowl, lavabo, wash-hand basin\n898:n04554684:washer, automatic washer, washing machine\n899:n04557648:water bottle\n900:n04560804:water jug\n901:n04562935:water tower\n902:n04579145:whiskey jug\n903:n04579432:whistle\n904:n04584207:wig\n905:n04589890:window screen\n906:n04590129:window shade\n907:n04591157:Windsor tie\n908:n04591713:wine bottle\n909:n04592741:wing\n910:n04596742:wok\n911:n04597913:wooden spoon\n912:n04599235:wool, woolen, woollen\n913:n04604644:worm fence, snake fence, snake-rail fence, Virginia fence\n914:n04606251:wreck\n915:n04612504:yawl\n916:n04613696:yurt\n917:n06359193:web site, website, internet site, site\n918:n06596364:comic book\n919:n06785654:crossword puzzle, crossword\n920:n06794110:street sign\n921:n06874185:traffic light, traffic signal, stoplight\n922:n07248320:book jacket, dust cover, dust jacket, dust wrapper\n923:n07565083:menu\n924:n07579787:plate\n925:n07583066:guacamole\n926:n07584110:consomme\n927:n07590611:hot pot, hotpot\n928:n07613480:trifle\n929:n07614500:ice cream, icecream\n930:n07615774:ice lolly, lolly, lollipop, popsicle\n931:n07684084:French loaf\n932:n07693725:bagel, beigel\n933:n07695742:pretzel\n934:n07697313:cheeseburger\n935:n07697537:hotdog, hot dog, red hot\n936:n07711569:mashed potato\n937:n07714571:head cabbage\n938:n07714990:broccoli\n939:n07715103:cauliflower\n940:n07716358:zucchini, courgette\n941:n07716906:spaghetti squash\n942:n07717410:acorn squash\n943:n07717556:butternut squash\n944:n07718472:cucumber, cuke\n945:n07718747:artichoke, globe artichoke\n946:n07720875:bell pepper\n947:n07730033:cardoon\n948:n07734744:mushroom\n949:n07742313:Granny Smith\n950:n07745940:strawberry\n951:n07747607:orange\n952:n07749582:lemon\n953:n07753113:fig\n954:n07753275:pineapple, ananas\n955:n07753592:banana\n956:n07754684:jackfruit, jak, jack\n957:n07760859:custard apple\n958:n07768694:pomegranate\n959:n07802026:hay\n960:n07831146:carbonara\n961:n07836838:chocolate sauce, chocolate syrup\n962:n07860988:dough\n963:n07871810:meat loaf, meatloaf\n964:n07873807:pizza, pizza pie\n965:n07875152:potpie\n966:n07880968:burrito\n967:n07892512:red wine\n968:n07920052:espresso\n969:n07930864:cup\n970:n07932039:eggnog\n971:n09193705:alp\n972:n09229709:bubble\n973:n09246464:cliff, drop, drop-off\n974:n09256479:coral reef\n975:n09288635:geyser\n976:n09332890:lakeside, lakeshore\n977:n09399592:promontory, headland, head, foreland\n978:n09421951:sandbar, sand bar\n979:n09428293:seashore, coast, seacoast, sea-coast\n980:n09468604:valley, vale\n981:n09472597:volcano\n982:n09835506:ballplayer, baseball player\n983:n10148035:groom, bridegroom\n984:n10565667:scuba diver\n985:n11879895:rapeseed\n986:n11939491:daisy\n987:n12057211:yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum\n988:n12144580:corn\n989:n12267677:acorn\n990:n12620546:hip, rose hip, rosehip\n991:n12768682:buckeye, horse chestnut, conker\n992:n12985857:coral fungus\n993:n12998815:agaric\n994:n13037406:gyromitra\n995:n13040303:stinkhorn, carrion fungus\n996:n13044778:earthstar\n997:n13052670:hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa\n998:n13054560:bolete\n999:n13133613:ear, spike, capitulum\n1000:n15075141:toilet tissue, toilet paper, bathroom tissue\n" }, { "path": "labels/labels.py", "content": "#!/usr/bin/env python3\n\nnew_label = \"labels.txt\"\nimagenet_map = \"synset_words.txt\"\nlabel2num = \"label2num.txt\"\n\nnum_dict = {}\n\nout_file = open(label2num, 'w')\n\nwith open(new_label, 'r') as f:\n for line in f:\n if line:\n num = int(line.split(':')[0])\n lab = line.split(':')[1].strip()\n num_dict[lab] = num\n\nprint(len(num_dict))\n\nwith open(imagenet_map, 'r') as f:\n for line in f:\n if line:\n nn = line[:9]\n lab = line[10:].strip()\n print(nn, lab)\n num = num_dict[lab]\n out_file.write(\"{}:{}:{}\\n\".format(num, nn, lab))\n\n" }, { "path": "labels/labels.txt", "content": "0:background\n1:tench, Tinca tinca\n2:goldfish, Carassius auratus\n3:great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias\n4:tiger shark, Galeocerdo cuvieri\n5:hammerhead, hammerhead shark\n6:electric ray, crampfish, numbfish, torpedo\n7:stingray\n8:cock\n9:hen\n10:ostrich, Struthio camelus\n11:brambling, Fringilla montifringilla\n12:goldfinch, Carduelis carduelis\n13:house finch, linnet, Carpodacus mexicanus\n14:junco, snowbird\n15:indigo bunting, indigo finch, indigo bird, Passerina cyanea\n16:robin, American robin, Turdus migratorius\n17:bulbul\n18:jay\n19:magpie\n20:chickadee\n21:water ouzel, dipper\n22:kite\n23:bald eagle, American eagle, Haliaeetus leucocephalus\n24:vulture\n25:great grey owl, great gray owl, Strix nebulosa\n26:European fire salamander, Salamandra salamandra\n27:common newt, Triturus vulgaris\n28:eft\n29:spotted salamander, Ambystoma maculatum\n30:axolotl, mud puppy, Ambystoma mexicanum\n31:bullfrog, Rana catesbeiana\n32:tree frog, tree-frog\n33:tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui\n34:loggerhead, loggerhead turtle, Caretta caretta\n35:leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea\n36:mud turtle\n37:terrapin\n38:box turtle, box tortoise\n39:banded gecko\n40:common iguana, iguana, Iguana iguana\n41:American chameleon, anole, Anolis carolinensis\n42:whiptail, whiptail lizard\n43:agama\n44:frilled lizard, Chlamydosaurus kingi\n45:alligator lizard\n46:Gila monster, Heloderma suspectum\n47:green lizard, Lacerta viridis\n48:African chameleon, Chamaeleo chamaeleon\n49:Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis\n50:African crocodile, Nile crocodile, Crocodylus niloticus\n51:American alligator, Alligator mississipiensis\n52:triceratops\n53:thunder snake, worm snake, Carphophis amoenus\n54:ringneck snake, ring-necked snake, ring snake\n55:hognose snake, puff adder, sand viper\n56:green snake, grass snake\n57:king snake, kingsnake\n58:garter snake, grass snake\n59:water snake\n60:vine snake\n61:night snake, Hypsiglena torquata\n62:boa constrictor, Constrictor constrictor\n63:rock python, rock snake, Python sebae\n64:Indian cobra, Naja naja\n65:green mamba\n66:sea snake\n67:horned viper, cerastes, sand viper, horned asp, Cerastes cornutus\n68:diamondback, diamondback rattlesnake, Crotalus adamanteus\n69:sidewinder, horned rattlesnake, Crotalus cerastes\n70:trilobite\n71:harvestman, daddy longlegs, Phalangium opilio\n72:scorpion\n73:black and gold garden spider, Argiope aurantia\n74:barn spider, Araneus cavaticus\n75:garden spider, Aranea diademata\n76:black widow, Latrodectus mactans\n77:tarantula\n78:wolf spider, hunting spider\n79:tick\n80:centipede\n81:black grouse\n82:ptarmigan\n83:ruffed grouse, partridge, Bonasa umbellus\n84:prairie chicken, prairie grouse, prairie fowl\n85:peacock\n86:quail\n87:partridge\n88:African grey, African gray, Psittacus erithacus\n89:macaw\n90:sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita\n91:lorikeet\n92:coucal\n93:bee eater\n94:hornbill\n95:hummingbird\n96:jacamar\n97:toucan\n98:drake\n99:red-breasted merganser, Mergus serrator\n100:goose\n101:black swan, Cygnus atratus\n102:tusker\n103:echidna, spiny anteater, anteater\n104:platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus\n105:wallaby, brush kangaroo\n106:koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus\n107:wombat\n108:jellyfish\n109:sea anemone, anemone\n110:brain coral\n111:flatworm, platyhelminth\n112:nematode, nematode worm, roundworm\n113:conch\n114:snail\n115:slug\n116:sea slug, nudibranch\n117:chiton, coat-of-mail shell, sea cradle, polyplacophore\n118:chambered nautilus, pearly nautilus, nautilus\n119:Dungeness crab, Cancer magister\n120:rock crab, Cancer irroratus\n121:fiddler crab\n122:king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica\n123:American lobster, Northern lobster, Maine lobster, Homarus americanus\n124:spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish\n125:crayfish, crawfish, crawdad, crawdaddy\n126:hermit crab\n127:isopod\n128:white stork, Ciconia ciconia\n129:black stork, Ciconia nigra\n130:spoonbill\n131:flamingo\n132:little blue heron, Egretta caerulea\n133:American egret, great white heron, Egretta albus\n134:bittern\n135:crane\n136:limpkin, Aramus pictus\n137:European gallinule, Porphyrio porphyrio\n138:American coot, marsh hen, mud hen, water hen, Fulica americana\n139:bustard\n140:ruddy turnstone, Arenaria interpres\n141:red-backed sandpiper, dunlin, Erolia alpina\n142:redshank, Tringa totanus\n143:dowitcher\n144:oystercatcher, oyster catcher\n145:pelican\n146:king penguin, Aptenodytes patagonica\n147:albatross, mollymawk\n148:grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus\n149:killer whale, killer, orca, grampus, sea wolf, Orcinus orca\n150:dugong, Dugong dugon\n151:sea lion\n152:Chihuahua\n153:Japanese spaniel\n154:Maltese dog, Maltese terrier, Maltese\n155:Pekinese, Pekingese, Peke\n156:Shih-Tzu\n157:Blenheim spaniel\n158:papillon\n159:toy terrier\n160:Rhodesian ridgeback\n161:Afghan hound, Afghan\n162:basset, basset hound\n163:beagle\n164:bloodhound, sleuthhound\n165:bluetick\n166:black-and-tan coonhound\n167:Walker hound, Walker foxhound\n168:English foxhound\n169:redbone\n170:borzoi, Russian wolfhound\n171:Irish wolfhound\n172:Italian greyhound\n173:whippet\n174:Ibizan hound, Ibizan Podenco\n175:Norwegian elkhound, elkhound\n176:otterhound, otter hound\n177:Saluki, gazelle hound\n178:Scottish deerhound, deerhound\n179:Weimaraner\n180:Staffordshire bullterrier, Staffordshire bull terrier\n181:American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier\n182:Bedlington terrier\n183:Border terrier\n184:Kerry blue terrier\n185:Irish terrier\n186:Norfolk terrier\n187:Norwich terrier\n188:Yorkshire terrier\n189:wire-haired fox terrier\n190:Lakeland terrier\n191:Sealyham terrier, Sealyham\n192:Airedale, Airedale terrier\n193:cairn, cairn terrier\n194:Australian terrier\n195:Dandie Dinmont, Dandie Dinmont terrier\n196:Boston bull, Boston terrier\n197:miniature schnauzer\n198:giant schnauzer\n199:standard schnauzer\n200:Scotch terrier, Scottish terrier, Scottie\n201:Tibetan terrier, chrysanthemum dog\n202:silky terrier, Sydney silky\n203:soft-coated wheaten terrier\n204:West Highland white terrier\n205:Lhasa, Lhasa apso\n206:flat-coated retriever\n207:curly-coated retriever\n208:golden retriever\n209:Labrador retriever\n210:Chesapeake Bay retriever\n211:German short-haired pointer\n212:vizsla, Hungarian pointer\n213:English setter\n214:Irish setter, red setter\n215:Gordon setter\n216:Brittany spaniel\n217:clumber, clumber spaniel\n218:English springer, English springer spaniel\n219:Welsh springer spaniel\n220:cocker spaniel, English cocker spaniel, cocker\n221:Sussex spaniel\n222:Irish water spaniel\n223:kuvasz\n224:schipperke\n225:groenendael\n226:malinois\n227:briard\n228:kelpie\n229:komondor\n230:Old English sheepdog, bobtail\n231:Shetland sheepdog, Shetland sheep dog, Shetland\n232:collie\n233:Border collie\n234:Bouvier des Flandres, Bouviers des Flandres\n235:Rottweiler\n236:German shepherd, German shepherd dog, German police dog, alsatian\n237:Doberman, Doberman pinscher\n238:miniature pinscher\n239:Greater Swiss Mountain dog\n240:Bernese mountain dog\n241:Appenzeller\n242:EntleBucher\n243:boxer\n244:bull mastiff\n245:Tibetan mastiff\n246:French bulldog\n247:Great Dane\n248:Saint Bernard, St Bernard\n249:Eskimo dog, husky\n250:malamute, malemute, Alaskan malamute\n251:Siberian husky\n252:dalmatian, coach dog, carriage dog\n253:affenpinscher, monkey pinscher, monkey dog\n254:basenji\n255:pug, pug-dog\n256:Leonberg\n257:Newfoundland, Newfoundland dog\n258:Great Pyrenees\n259:Samoyed, Samoyede\n260:Pomeranian\n261:chow, chow chow\n262:keeshond\n263:Brabancon griffon\n264:Pembroke, Pembroke Welsh corgi\n265:Cardigan, Cardigan Welsh corgi\n266:toy poodle\n267:miniature poodle\n268:standard poodle\n269:Mexican hairless\n270:timber wolf, grey wolf, gray wolf, Canis lupus\n271:white wolf, Arctic wolf, Canis lupus tundrarum\n272:red wolf, maned wolf, Canis rufus, Canis niger\n273:coyote, prairie wolf, brush wolf, Canis latrans\n274:dingo, warrigal, warragal, Canis dingo\n275:dhole, Cuon alpinus\n276:African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus\n277:hyena, hyaena\n278:red fox, Vulpes vulpes\n279:kit fox, Vulpes macrotis\n280:Arctic fox, white fox, Alopex lagopus\n281:grey fox, gray fox, Urocyon cinereoargenteus\n282:tabby, tabby cat\n283:tiger cat\n284:Persian cat\n285:Siamese cat, Siamese\n286:Egyptian cat\n287:cougar, puma, catamount, mountain lion, painter, panther, Felis concolor\n288:lynx, catamount\n289:leopard, Panthera pardus\n290:snow leopard, ounce, Panthera uncia\n291:jaguar, panther, Panthera onca, Felis onca\n292:lion, king of beasts, Panthera leo\n293:tiger, Panthera tigris\n294:cheetah, chetah, Acinonyx jubatus\n295:brown bear, bruin, Ursus arctos\n296:American black bear, black bear, Ursus americanus, Euarctos americanus\n297:ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus\n298:sloth bear, Melursus ursinus, Ursus ursinus\n299:mongoose\n300:meerkat, mierkat\n301:tiger beetle\n302:ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle\n303:ground beetle, carabid beetle\n304:long-horned beetle, longicorn, longicorn beetle\n305:leaf beetle, chrysomelid\n306:dung beetle\n307:rhinoceros beetle\n308:weevil\n309:fly\n310:bee\n311:ant, emmet, pismire\n312:grasshopper, hopper\n313:cricket\n314:walking stick, walkingstick, stick insect\n315:cockroach, roach\n316:mantis, mantid\n317:cicada, cicala\n318:leafhopper\n319:lacewing, lacewing fly\n320:dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk\n321:damselfly\n322:admiral\n323:ringlet, ringlet butterfly\n324:monarch, monarch butterfly, milkweed butterfly, Danaus plexippus\n325:cabbage butterfly\n326:sulphur butterfly, sulfur butterfly\n327:lycaenid, lycaenid butterfly\n328:starfish, sea star\n329:sea urchin\n330:sea cucumber, holothurian\n331:wood rabbit, cottontail, cottontail rabbit\n332:hare\n333:Angora, Angora rabbit\n334:hamster\n335:porcupine, hedgehog\n336:fox squirrel, eastern fox squirrel, Sciurus niger\n337:marmot\n338:beaver\n339:guinea pig, Cavia cobaya\n340:sorrel\n341:zebra\n342:hog, pig, grunter, squealer, Sus scrofa\n343:wild boar, boar, Sus scrofa\n344:warthog\n345:hippopotamus, hippo, river horse, Hippopotamus amphibius\n346:ox\n347:water buffalo, water ox, Asiatic buffalo, Bubalus bubalis\n348:bison\n349:ram, tup\n350:bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis\n351:ibex, Capra ibex\n352:hartebeest\n353:impala, Aepyceros melampus\n354:gazelle\n355:Arabian camel, dromedary, Camelus dromedarius\n356:llama\n357:weasel\n358:mink\n359:polecat, fitch, foulmart, foumart, Mustela putorius\n360:black-footed ferret, ferret, Mustela nigripes\n361:otter\n362:skunk, polecat, wood pussy\n363:badger\n364:armadillo\n365:three-toed sloth, ai, Bradypus tridactylus\n366:orangutan, orang, orangutang, Pongo pygmaeus\n367:gorilla, Gorilla gorilla\n368:chimpanzee, chimp, Pan troglodytes\n369:gibbon, Hylobates lar\n370:siamang, Hylobates syndactylus, Symphalangus syndactylus\n371:guenon, guenon monkey\n372:patas, hussar monkey, Erythrocebus patas\n373:baboon\n374:macaque\n375:langur\n376:colobus, colobus monkey\n377:proboscis monkey, Nasalis larvatus\n378:marmoset\n379:capuchin, ringtail, Cebus capucinus\n380:howler monkey, howler\n381:titi, titi monkey\n382:spider monkey, Ateles geoffroyi\n383:squirrel monkey, Saimiri sciureus\n384:Madagascar cat, ring-tailed lemur, Lemur catta\n385:indri, indris, Indri indri, Indri brevicaudatus\n386:Indian elephant, Elephas maximus\n387:African elephant, Loxodonta africana\n388:lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens\n389:giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca\n390:barracouta, snoek\n391:eel\n392:coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch\n393:rock beauty, Holocanthus tricolor\n394:anemone fish\n395:sturgeon\n396:gar, garfish, garpike, billfish, Lepisosteus osseus\n397:lionfish\n398:puffer, pufferfish, blowfish, globefish\n399:abacus\n400:abaya\n401:academic gown, academic robe, judge's robe\n402:accordion, piano accordion, squeeze box\n403:acoustic guitar\n404:aircraft carrier, carrier, flattop, attack aircraft carrier\n405:airliner\n406:airship, dirigible\n407:altar\n408:ambulance\n409:amphibian, amphibious vehicle\n410:analog clock\n411:apiary, bee house\n412:apron\n413:ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin\n414:assault rifle, assault gun\n415:backpack, back pack, knapsack, packsack, rucksack, haversack\n416:bakery, bakeshop, bakehouse\n417:balance beam, beam\n418:balloon\n419:ballpoint, ballpoint pen, ballpen, Biro\n420:Band Aid\n421:banjo\n422:bannister, banister, balustrade, balusters, handrail\n423:barbell\n424:barber chair\n425:barbershop\n426:barn\n427:barometer\n428:barrel, cask\n429:barrow, garden cart, lawn cart, wheelbarrow\n430:baseball\n431:basketball\n432:bassinet\n433:bassoon\n434:bathing cap, swimming cap\n435:bath towel\n436:bathtub, bathing tub, bath, tub\n437:beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon\n438:beacon, lighthouse, beacon light, pharos\n439:beaker\n440:bearskin, busby, shako\n441:beer bottle\n442:beer glass\n443:bell cote, bell cot\n444:bib\n445:bicycle-built-for-two, tandem bicycle, tandem\n446:bikini, two-piece\n447:binder, ring-binder\n448:binoculars, field glasses, opera glasses\n449:birdhouse\n450:boathouse\n451:bobsled, bobsleigh, bob\n452:bolo tie, bolo, bola tie, bola\n453:bonnet, poke bonnet\n454:bookcase\n455:bookshop, bookstore, bookstall\n456:bottlecap\n457:bow\n458:bow tie, bow-tie, bowtie\n459:brass, memorial tablet, plaque\n460:brassiere, bra, bandeau\n461:breakwater, groin, groyne, mole, bulwark, seawall, jetty\n462:breastplate, aegis, egis\n463:broom\n464:bucket, pail\n465:buckle\n466:bulletproof vest\n467:bullet train, bullet\n468:butcher shop, meat market\n469:cab, hack, taxi, taxicab\n470:caldron, cauldron\n471:candle, taper, wax light\n472:cannon\n473:canoe\n474:can opener, tin opener\n475:cardigan\n476:car mirror\n477:carousel, carrousel, merry-go-round, roundabout, whirligig\n478:carpenter's kit, tool kit\n479:carton\n480:car wheel\n481:cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM\n482:cassette\n483:cassette player\n484:castle\n485:catamaran\n486:CD player\n487:cello, violoncello\n488:cellular telephone, cellular phone, cellphone, cell, mobile phone\n489:chain\n490:chainlink fence\n491:chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour\n492:chain saw, chainsaw\n493:chest\n494:chiffonier, commode\n495:chime, bell, gong\n496:china cabinet, china closet\n497:Christmas stocking\n498:church, church building\n499:cinema, movie theater, movie theatre, movie house, picture palace\n500:cleaver, meat cleaver, chopper\n501:cliff dwelling\n502:cloak\n503:clog, geta, patten, sabot\n504:cocktail shaker\n505:coffee mug\n506:coffeepot\n507:coil, spiral, volute, whorl, helix\n508:combination lock\n509:computer keyboard, keypad\n510:confectionery, confectionary, candy store\n511:container ship, containership, container vessel\n512:convertible\n513:corkscrew, bottle screw\n514:cornet, horn, trumpet, trump\n515:cowboy boot\n516:cowboy hat, ten-gallon hat\n517:cradle\n518:crane\n519:crash helmet\n520:crate\n521:crib, cot\n522:Crock Pot\n523:croquet ball\n524:crutch\n525:cuirass\n526:dam, dike, dyke\n527:desk\n528:desktop computer\n529:dial telephone, dial phone\n530:diaper, nappy, napkin\n531:digital clock\n532:digital watch\n533:dining table, board\n534:dishrag, dishcloth\n535:dishwasher, dish washer, dishwashing machine\n536:disk brake, disc brake\n537:dock, dockage, docking facility\n538:dogsled, dog sled, dog sleigh\n539:dome\n540:doormat, welcome mat\n541:drilling platform, offshore rig\n542:drum, membranophone, tympan\n543:drumstick\n544:dumbbell\n545:Dutch oven\n546:electric fan, blower\n547:electric guitar\n548:electric locomotive\n549:entertainment center\n550:envelope\n551:espresso maker\n552:face powder\n553:feather boa, boa\n554:file, file cabinet, filing cabinet\n555:fireboat\n556:fire engine, fire truck\n557:fire screen, fireguard\n558:flagpole, flagstaff\n559:flute, transverse flute\n560:folding chair\n561:football helmet\n562:forklift\n563:fountain\n564:fountain pen\n565:four-poster\n566:freight car\n567:French horn, horn\n568:frying pan, frypan, skillet\n569:fur coat\n570:garbage truck, dustcart\n571:gasmask, respirator, gas helmet\n572:gas pump, gasoline pump, petrol pump, island dispenser\n573:goblet\n574:go-kart\n575:golf ball\n576:golfcart, golf cart\n577:gondola\n578:gong, tam-tam\n579:gown\n580:grand piano, grand\n581:greenhouse, nursery, glasshouse\n582:grille, radiator grille\n583:grocery store, grocery, food market, market\n584:guillotine\n585:hair slide\n586:hair spray\n587:half track\n588:hammer\n589:hamper\n590:hand blower, blow dryer, blow drier, hair dryer, hair drier\n591:hand-held computer, hand-held microcomputer\n592:handkerchief, hankie, hanky, hankey\n593:hard disc, hard disk, fixed disk\n594:harmonica, mouth organ, harp, mouth harp\n595:harp\n596:harvester, reaper\n597:hatchet\n598:holster\n599:home theater, home theatre\n600:honeycomb\n601:hook, claw\n602:hoopskirt, crinoline\n603:horizontal bar, high bar\n604:horse cart, horse-cart\n605:hourglass\n606:iPod\n607:iron, smoothing iron\n608:jack-o'-lantern\n609:jean, blue jean, denim\n610:jeep, landrover\n611:jersey, T-shirt, tee shirt\n612:jigsaw puzzle\n613:jinrikisha, ricksha, rickshaw\n614:joystick\n615:kimono\n616:knee pad\n617:knot\n618:lab coat, laboratory coat\n619:ladle\n620:lampshade, lamp shade\n621:laptop, laptop computer\n622:lawn mower, mower\n623:lens cap, lens cover\n624:letter opener, paper knife, paperknife\n625:library\n626:lifeboat\n627:lighter, light, igniter, ignitor\n628:limousine, limo\n629:liner, ocean liner\n630:lipstick, lip rouge\n631:Loafer\n632:lotion\n633:loudspeaker, speaker, speaker unit, loudspeaker system, speaker system\n634:loupe, jeweler's loupe\n635:lumbermill, sawmill\n636:magnetic compass\n637:mailbag, postbag\n638:mailbox, letter box\n639:maillot\n640:maillot, tank suit\n641:manhole cover\n642:maraca\n643:marimba, xylophone\n644:mask\n645:matchstick\n646:maypole\n647:maze, labyrinth\n648:measuring cup\n649:medicine chest, medicine cabinet\n650:megalith, megalithic structure\n651:microphone, mike\n652:microwave, microwave oven\n653:military uniform\n654:milk can\n655:minibus\n656:miniskirt, mini\n657:minivan\n658:missile\n659:mitten\n660:mixing bowl\n661:mobile home, manufactured home\n662:Model T\n663:modem\n664:monastery\n665:monitor\n666:moped\n667:mortar\n668:mortarboard\n669:mosque\n670:mosquito net\n671:motor scooter, scooter\n672:mountain bike, all-terrain bike, off-roader\n673:mountain tent\n674:mouse, computer mouse\n675:mousetrap\n676:moving van\n677:muzzle\n678:nail\n679:neck brace\n680:necklace\n681:nipple\n682:notebook, notebook computer\n683:obelisk\n684:oboe, hautboy, hautbois\n685:ocarina, sweet potato\n686:odometer, hodometer, mileometer, milometer\n687:oil filter\n688:organ, pipe organ\n689:oscilloscope, scope, cathode-ray oscilloscope, CRO\n690:overskirt\n691:oxcart\n692:oxygen mask\n693:packet\n694:paddle, boat paddle\n695:paddlewheel, paddle wheel\n696:padlock\n697:paintbrush\n698:pajama, pyjama, pj's, jammies\n699:palace\n700:panpipe, pandean pipe, syrinx\n701:paper towel\n702:parachute, chute\n703:parallel bars, bars\n704:park bench\n705:parking meter\n706:passenger car, coach, carriage\n707:patio, terrace\n708:pay-phone, pay-station\n709:pedestal, plinth, footstall\n710:pencil box, pencil case\n711:pencil sharpener\n712:perfume, essence\n713:Petri dish\n714:photocopier\n715:pick, plectrum, plectron\n716:pickelhaube\n717:picket fence, paling\n718:pickup, pickup truck\n719:pier\n720:piggy bank, penny bank\n721:pill bottle\n722:pillow\n723:ping-pong ball\n724:pinwheel\n725:pirate, pirate ship\n726:pitcher, ewer\n727:plane, carpenter's plane, woodworking plane\n728:planetarium\n729:plastic bag\n730:plate rack\n731:plow, plough\n732:plunger, plumber's helper\n733:Polaroid camera, Polaroid Land camera\n734:pole\n735:police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria\n736:poncho\n737:pool table, billiard table, snooker table\n738:pop bottle, soda bottle\n739:pot, flowerpot\n740:potter's wheel\n741:power drill\n742:prayer rug, prayer mat\n743:printer\n744:prison, prison house\n745:projectile, missile\n746:projector\n747:puck, hockey puck\n748:punching bag, punch bag, punching ball, punchball\n749:purse\n750:quill, quill pen\n751:quilt, comforter, comfort, puff\n752:racer, race car, racing car\n753:racket, racquet\n754:radiator\n755:radio, wireless\n756:radio telescope, radio reflector\n757:rain barrel\n758:recreational vehicle, RV, R.V.\n759:reel\n760:reflex camera\n761:refrigerator, icebox\n762:remote control, remote\n763:restaurant, eating house, eating place, eatery\n764:revolver, six-gun, six-shooter\n765:rifle\n766:rocking chair, rocker\n767:rotisserie\n768:rubber eraser, rubber, pencil eraser\n769:rugby ball\n770:rule, ruler\n771:running shoe\n772:safe\n773:safety pin\n774:saltshaker, salt shaker\n775:sandal\n776:sarong\n777:sax, saxophone\n778:scabbard\n779:scale, weighing machine\n780:school bus\n781:schooner\n782:scoreboard\n783:screen, CRT screen\n784:screw\n785:screwdriver\n786:seat belt, seatbelt\n787:sewing machine\n788:shield, buckler\n789:shoe shop, shoe-shop, shoe store\n790:shoji\n791:shopping basket\n792:shopping cart\n793:shovel\n794:shower cap\n795:shower curtain\n796:ski\n797:ski mask\n798:sleeping bag\n799:slide rule, slipstick\n800:sliding door\n801:slot, one-armed bandit\n802:snorkel\n803:snowmobile\n804:snowplow, snowplough\n805:soap dispenser\n806:soccer ball\n807:sock\n808:solar dish, solar collector, solar furnace\n809:sombrero\n810:soup bowl\n811:space bar\n812:space heater\n813:space shuttle\n814:spatula\n815:speedboat\n816:spider web, spider's web\n817:spindle\n818:sports car, sport car\n819:spotlight, spot\n820:stage\n821:steam locomotive\n822:steel arch bridge\n823:steel drum\n824:stethoscope\n825:stole\n826:stone wall\n827:stopwatch, stop watch\n828:stove\n829:strainer\n830:streetcar, tram, tramcar, trolley, trolley car\n831:stretcher\n832:studio couch, day bed\n833:stupa, tope\n834:submarine, pigboat, sub, U-boat\n835:suit, suit of clothes\n836:sundial\n837:sunglass\n838:sunglasses, dark glasses, shades\n839:sunscreen, sunblock, sun blocker\n840:suspension bridge\n841:swab, swob, mop\n842:sweatshirt\n843:swimming trunks, bathing trunks\n844:swing\n845:switch, electric switch, electrical switch\n846:syringe\n847:table lamp\n848:tank, army tank, armored combat vehicle, armoured combat vehicle\n849:tape player\n850:teapot\n851:teddy, teddy bear\n852:television, television system\n853:tennis ball\n854:thatch, thatched roof\n855:theater curtain, theatre curtain\n856:thimble\n857:thresher, thrasher, threshing machine\n858:throne\n859:tile roof\n860:toaster\n861:tobacco shop, tobacconist shop, tobacconist\n862:toilet seat\n863:torch\n864:totem pole\n865:tow truck, tow car, wrecker\n866:toyshop\n867:tractor\n868:trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi\n869:tray\n870:trench coat\n871:tricycle, trike, velocipede\n872:trimaran\n873:tripod\n874:triumphal arch\n875:trolleybus, trolley coach, trackless trolley\n876:trombone\n877:tub, vat\n878:turnstile\n879:typewriter keyboard\n880:umbrella\n881:unicycle, monocycle\n882:upright, upright piano\n883:vacuum, vacuum cleaner\n884:vase\n885:vault\n886:velvet\n887:vending machine\n888:vestment\n889:viaduct\n890:violin, fiddle\n891:volleyball\n892:waffle iron\n893:wall clock\n894:wallet, billfold, notecase, pocketbook\n895:wardrobe, closet, press\n896:warplane, military plane\n897:washbasin, handbasin, washbowl, lavabo, wash-hand basin\n898:washer, automatic washer, washing machine\n899:water bottle\n900:water jug\n901:water tower\n902:whiskey jug\n903:whistle\n904:wig\n905:window screen\n906:window shade\n907:Windsor tie\n908:wine bottle\n909:wing\n910:wok\n911:wooden spoon\n912:wool, woolen, woollen\n913:worm fence, snake fence, snake-rail fence, Virginia fence\n914:wreck\n915:yawl\n916:yurt\n917:web site, website, internet site, site\n918:comic book\n919:crossword puzzle, crossword\n920:street sign\n921:traffic light, traffic signal, stoplight\n922:book jacket, dust cover, dust jacket, dust wrapper\n923:menu\n924:plate\n925:guacamole\n926:consomme\n927:hot pot, hotpot\n928:trifle\n929:ice cream, icecream\n930:ice lolly, lolly, lollipop, popsicle\n931:French loaf\n932:bagel, beigel\n933:pretzel\n934:cheeseburger\n935:hotdog, hot dog, red hot\n936:mashed potato\n937:head cabbage\n938:broccoli\n939:cauliflower\n940:zucchini, courgette\n941:spaghetti squash\n942:acorn squash\n943:butternut squash\n944:cucumber, cuke\n945:artichoke, globe artichoke\n946:bell pepper\n947:cardoon\n948:mushroom\n949:Granny Smith\n950:strawberry\n951:orange\n952:lemon\n953:fig\n954:pineapple, ananas\n955:banana\n956:jackfruit, jak, jack\n957:custard apple\n958:pomegranate\n959:hay\n960:carbonara\n961:chocolate sauce, chocolate syrup\n962:dough\n963:meat loaf, meatloaf\n964:pizza, pizza pie\n965:potpie\n966:burrito\n967:red wine\n968:espresso\n969:cup\n970:eggnog\n971:alp\n972:bubble\n973:cliff, drop, drop-off\n974:coral reef\n975:geyser\n976:lakeside, lakeshore\n977:promontory, headland, head, foreland\n978:sandbar, sand bar\n979:seashore, coast, seacoast, sea-coast\n980:valley, vale\n981:volcano\n982:ballplayer, baseball player\n983:groom, bridegroom\n984:scuba diver\n985:rapeseed\n986:daisy\n987:yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum\n988:corn\n989:acorn\n990:hip, rose hip, rosehip\n991:buckeye, horse chestnut, conker\n992:coral fungus\n993:agaric\n994:gyromitra\n995:stinkhorn, carrion fungus\n996:earthstar\n997:hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa\n998:bolete\n999:ear, spike, capitulum\n1000:toilet tissue, toilet paper, bathroom tissue\n" }, { "path": "labels/synset_words.txt", "content": "n01440764 tench, Tinca tinca\nn01443537 goldfish, Carassius auratus\nn01484850 great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias\nn01491361 tiger shark, Galeocerdo cuvieri\nn01494475 hammerhead, hammerhead shark\nn01496331 electric ray, crampfish, numbfish, torpedo\nn01498041 stingray\nn01514668 cock\nn01514859 hen\nn01518878 ostrich, Struthio camelus\nn01530575 brambling, Fringilla montifringilla\nn01531178 goldfinch, Carduelis carduelis\nn01532829 house finch, linnet, Carpodacus mexicanus\nn01534433 junco, snowbird\nn01537544 indigo bunting, indigo finch, indigo bird, Passerina cyanea\nn01558993 robin, American robin, Turdus migratorius\nn01560419 bulbul\nn01580077 jay\nn01582220 magpie\nn01592084 chickadee\nn01601694 water ouzel, dipper\nn01608432 kite\nn01614925 bald eagle, American eagle, Haliaeetus leucocephalus\nn01616318 vulture\nn01622779 great grey owl, great gray owl, Strix nebulosa\nn01629819 European fire salamander, Salamandra salamandra\nn01630670 common newt, Triturus vulgaris\nn01631663 eft\nn01632458 spotted salamander, Ambystoma maculatum\nn01632777 axolotl, mud puppy, Ambystoma mexicanum\nn01641577 bullfrog, Rana catesbeiana\nn01644373 tree frog, tree-frog\nn01644900 tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui\nn01664065 loggerhead, loggerhead turtle, Caretta caretta\nn01665541 leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea\nn01667114 mud turtle\nn01667778 terrapin\nn01669191 box turtle, box tortoise\nn01675722 banded gecko\nn01677366 common iguana, iguana, Iguana iguana\nn01682714 American chameleon, anole, Anolis carolinensis\nn01685808 whiptail, whiptail lizard\nn01687978 agama\nn01688243 frilled lizard, Chlamydosaurus kingi\nn01689811 alligator lizard\nn01692333 Gila monster, Heloderma suspectum\nn01693334 green lizard, Lacerta viridis\nn01694178 African chameleon, Chamaeleo chamaeleon\nn01695060 Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis\nn01697457 African crocodile, Nile crocodile, Crocodylus niloticus\nn01698640 American alligator, Alligator mississipiensis\nn01704323 triceratops\nn01728572 thunder snake, worm snake, Carphophis amoenus\nn01728920 ringneck snake, ring-necked snake, ring snake\nn01729322 hognose snake, puff adder, sand viper\nn01729977 green snake, grass snake\nn01734418 king snake, kingsnake\nn01735189 garter snake, grass snake\nn01737021 water snake\nn01739381 vine snake\nn01740131 night snake, Hypsiglena torquata\nn01742172 boa constrictor, Constrictor constrictor\nn01744401 rock python, rock snake, Python sebae\nn01748264 Indian cobra, Naja naja\nn01749939 green mamba\nn01751748 sea snake\nn01753488 horned viper, cerastes, sand viper, horned asp, Cerastes cornutus\nn01755581 diamondback, diamondback rattlesnake, Crotalus adamanteus\nn01756291 sidewinder, horned rattlesnake, Crotalus cerastes\nn01768244 trilobite\nn01770081 harvestman, daddy longlegs, Phalangium opilio\nn01770393 scorpion\nn01773157 black and gold garden spider, Argiope aurantia\nn01773549 barn spider, Araneus cavaticus\nn01773797 garden spider, Aranea diademata\nn01774384 black widow, Latrodectus mactans\nn01774750 tarantula\nn01775062 wolf spider, hunting spider\nn01776313 tick\nn01784675 centipede\nn01795545 black grouse\nn01796340 ptarmigan\nn01797886 ruffed grouse, partridge, Bonasa umbellus\nn01798484 prairie chicken, prairie grouse, prairie fowl\nn01806143 peacock\nn01806567 quail\nn01807496 partridge\nn01817953 African grey, African gray, Psittacus erithacus\nn01818515 macaw\nn01819313 sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita\nn01820546 lorikeet\nn01824575 coucal\nn01828970 bee eater\nn01829413 hornbill\nn01833805 hummingbird\nn01843065 jacamar\nn01843383 toucan\nn01847000 drake\nn01855032 red-breasted merganser, Mergus serrator\nn01855672 goose\nn01860187 black swan, Cygnus atratus\nn01871265 tusker\nn01872401 echidna, spiny anteater, anteater\nn01873310 platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus\nn01877812 wallaby, brush kangaroo\nn01882714 koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus\nn01883070 wombat\nn01910747 jellyfish\nn01914609 sea anemone, anemone\nn01917289 brain coral\nn01924916 flatworm, platyhelminth\nn01930112 nematode, nematode worm, roundworm\nn01943899 conch\nn01944390 snail\nn01945685 slug\nn01950731 sea slug, nudibranch\nn01955084 chiton, coat-of-mail shell, sea cradle, polyplacophore\nn01968897 chambered nautilus, pearly nautilus, nautilus\nn01978287 Dungeness crab, Cancer magister\nn01978455 rock crab, Cancer irroratus\nn01980166 fiddler crab\nn01981276 king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica\nn01983481 American lobster, Northern lobster, Maine lobster, Homarus americanus\nn01984695 spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish\nn01985128 crayfish, crawfish, crawdad, crawdaddy\nn01986214 hermit crab\nn01990800 isopod\nn02002556 white stork, Ciconia ciconia\nn02002724 black stork, Ciconia nigra\nn02006656 spoonbill\nn02007558 flamingo\nn02009229 little blue heron, Egretta caerulea\nn02009912 American egret, great white heron, Egretta albus\nn02011460 bittern\nn02012849 crane\nn02013706 limpkin, Aramus pictus\nn02017213 European gallinule, Porphyrio porphyrio\nn02018207 American coot, marsh hen, mud hen, water hen, Fulica americana\nn02018795 bustard\nn02025239 ruddy turnstone, Arenaria interpres\nn02027492 red-backed sandpiper, dunlin, Erolia alpina\nn02028035 redshank, Tringa totanus\nn02033041 dowitcher\nn02037110 oystercatcher, oyster catcher\nn02051845 pelican\nn02056570 king penguin, Aptenodytes patagonica\nn02058221 albatross, mollymawk\nn02066245 grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus\nn02071294 killer whale, killer, orca, grampus, sea wolf, Orcinus orca\nn02074367 dugong, Dugong dugon\nn02077923 sea lion\nn02085620 Chihuahua\nn02085782 Japanese spaniel\nn02085936 Maltese dog, Maltese terrier, Maltese\nn02086079 Pekinese, Pekingese, Peke\nn02086240 Shih-Tzu\nn02086646 Blenheim spaniel\nn02086910 papillon\nn02087046 toy terrier\nn02087394 Rhodesian ridgeback\nn02088094 Afghan hound, Afghan\nn02088238 basset, basset hound\nn02088364 beagle\nn02088466 bloodhound, sleuthhound\nn02088632 bluetick\nn02089078 black-and-tan coonhound\nn02089867 Walker hound, Walker foxhound\nn02089973 English foxhound\nn02090379 redbone\nn02090622 borzoi, Russian wolfhound\nn02090721 Irish wolfhound\nn02091032 Italian greyhound\nn02091134 whippet\nn02091244 Ibizan hound, Ibizan Podenco\nn02091467 Norwegian elkhound, elkhound\nn02091635 otterhound, otter hound\nn02091831 Saluki, gazelle hound\nn02092002 Scottish deerhound, deerhound\nn02092339 Weimaraner\nn02093256 Staffordshire bullterrier, Staffordshire bull terrier\nn02093428 American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier\nn02093647 Bedlington terrier\nn02093754 Border terrier\nn02093859 Kerry blue terrier\nn02093991 Irish terrier\nn02094114 Norfolk terrier\nn02094258 Norwich terrier\nn02094433 Yorkshire terrier\nn02095314 wire-haired fox terrier\nn02095570 Lakeland terrier\nn02095889 Sealyham terrier, Sealyham\nn02096051 Airedale, Airedale terrier\nn02096177 cairn, cairn terrier\nn02096294 Australian terrier\nn02096437 Dandie Dinmont, Dandie Dinmont terrier\nn02096585 Boston bull, Boston terrier\nn02097047 miniature schnauzer\nn02097130 giant schnauzer\nn02097209 standard schnauzer\nn02097298 Scotch terrier, Scottish terrier, Scottie\nn02097474 Tibetan terrier, chrysanthemum dog\nn02097658 silky terrier, Sydney silky\nn02098105 soft-coated wheaten terrier\nn02098286 West Highland white terrier\nn02098413 Lhasa, Lhasa apso\nn02099267 flat-coated retriever\nn02099429 curly-coated retriever\nn02099601 golden retriever\nn02099712 Labrador retriever\nn02099849 Chesapeake Bay retriever\nn02100236 German short-haired pointer\nn02100583 vizsla, Hungarian pointer\nn02100735 English setter\nn02100877 Irish setter, red setter\nn02101006 Gordon setter\nn02101388 Brittany spaniel\nn02101556 clumber, clumber spaniel\nn02102040 English springer, English springer spaniel\nn02102177 Welsh springer spaniel\nn02102318 cocker spaniel, English cocker spaniel, cocker\nn02102480 Sussex spaniel\nn02102973 Irish water spaniel\nn02104029 kuvasz\nn02104365 schipperke\nn02105056 groenendael\nn02105162 malinois\nn02105251 briard\nn02105412 kelpie\nn02105505 komondor\nn02105641 Old English sheepdog, bobtail\nn02105855 Shetland sheepdog, Shetland sheep dog, Shetland\nn02106030 collie\nn02106166 Border collie\nn02106382 Bouvier des Flandres, Bouviers des Flandres\nn02106550 Rottweiler\nn02106662 German shepherd, German shepherd dog, German police dog, alsatian\nn02107142 Doberman, Doberman pinscher\nn02107312 miniature pinscher\nn02107574 Greater Swiss Mountain dog\nn02107683 Bernese mountain dog\nn02107908 Appenzeller\nn02108000 EntleBucher\nn02108089 boxer\nn02108422 bull mastiff\nn02108551 Tibetan mastiff\nn02108915 French bulldog\nn02109047 Great Dane\nn02109525 Saint Bernard, St Bernard\nn02109961 Eskimo dog, husky\nn02110063 malamute, malemute, Alaskan malamute\nn02110185 Siberian husky\nn02110341 dalmatian, coach dog, carriage dog\nn02110627 affenpinscher, monkey pinscher, monkey dog\nn02110806 basenji\nn02110958 pug, pug-dog\nn02111129 Leonberg\nn02111277 Newfoundland, Newfoundland dog\nn02111500 Great Pyrenees\nn02111889 Samoyed, Samoyede\nn02112018 Pomeranian\nn02112137 chow, chow chow\nn02112350 keeshond\nn02112706 Brabancon griffon\nn02113023 Pembroke, Pembroke Welsh corgi\nn02113186 Cardigan, Cardigan Welsh corgi\nn02113624 toy poodle\nn02113712 miniature poodle\nn02113799 standard poodle\nn02113978 Mexican hairless\nn02114367 timber wolf, grey wolf, gray wolf, Canis lupus\nn02114548 white wolf, Arctic wolf, Canis lupus tundrarum\nn02114712 red wolf, maned wolf, Canis rufus, Canis niger\nn02114855 coyote, prairie wolf, brush wolf, Canis latrans\nn02115641 dingo, warrigal, warragal, Canis dingo\nn02115913 dhole, Cuon alpinus\nn02116738 African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus\nn02117135 hyena, hyaena\nn02119022 red fox, Vulpes vulpes\nn02119789 kit fox, Vulpes macrotis\nn02120079 Arctic fox, white fox, Alopex lagopus\nn02120505 grey fox, gray fox, Urocyon cinereoargenteus\nn02123045 tabby, tabby cat\nn02123159 tiger cat\nn02123394 Persian cat\nn02123597 Siamese cat, Siamese\nn02124075 Egyptian cat\nn02125311 cougar, puma, catamount, mountain lion, painter, panther, Felis concolor\nn02127052 lynx, catamount\nn02128385 leopard, Panthera pardus\nn02128757 snow leopard, ounce, Panthera uncia\nn02128925 jaguar, panther, Panthera onca, Felis onca\nn02129165 lion, king of beasts, Panthera leo\nn02129604 tiger, Panthera tigris\nn02130308 cheetah, chetah, Acinonyx jubatus\nn02132136 brown bear, bruin, Ursus arctos\nn02133161 American black bear, black bear, Ursus americanus, Euarctos americanus\nn02134084 ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus\nn02134418 sloth bear, Melursus ursinus, Ursus ursinus\nn02137549 mongoose\nn02138441 meerkat, mierkat\nn02165105 tiger beetle\nn02165456 ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle\nn02167151 ground beetle, carabid beetle\nn02168699 long-horned beetle, longicorn, longicorn beetle\nn02169497 leaf beetle, chrysomelid\nn02172182 dung beetle\nn02174001 rhinoceros beetle\nn02177972 weevil\nn02190166 fly\nn02206856 bee\nn02219486 ant, emmet, pismire\nn02226429 grasshopper, hopper\nn02229544 cricket\nn02231487 walking stick, walkingstick, stick insect\nn02233338 cockroach, roach\nn02236044 mantis, mantid\nn02256656 cicada, cicala\nn02259212 leafhopper\nn02264363 lacewing, lacewing fly\nn02268443 dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk\nn02268853 damselfly\nn02276258 admiral\nn02277742 ringlet, ringlet butterfly\nn02279972 monarch, monarch butterfly, milkweed butterfly, Danaus plexippus\nn02280649 cabbage butterfly\nn02281406 sulphur butterfly, sulfur butterfly\nn02281787 lycaenid, lycaenid butterfly\nn02317335 starfish, sea star\nn02319095 sea urchin\nn02321529 sea cucumber, holothurian\nn02325366 wood rabbit, cottontail, cottontail rabbit\nn02326432 hare\nn02328150 Angora, Angora rabbit\nn02342885 hamster\nn02346627 porcupine, hedgehog\nn02356798 fox squirrel, eastern fox squirrel, Sciurus niger\nn02361337 marmot\nn02363005 beaver\nn02364673 guinea pig, Cavia cobaya\nn02389026 sorrel\nn02391049 zebra\nn02395406 hog, pig, grunter, squealer, Sus scrofa\nn02396427 wild boar, boar, Sus scrofa\nn02397096 warthog\nn02398521 hippopotamus, hippo, river horse, Hippopotamus amphibius\nn02403003 ox\nn02408429 water buffalo, water ox, Asiatic buffalo, Bubalus bubalis\nn02410509 bison\nn02412080 ram, tup\nn02415577 bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis\nn02417914 ibex, Capra ibex\nn02422106 hartebeest\nn02422699 impala, Aepyceros melampus\nn02423022 gazelle\nn02437312 Arabian camel, dromedary, Camelus dromedarius\nn02437616 llama\nn02441942 weasel\nn02442845 mink\nn02443114 polecat, fitch, foulmart, foumart, Mustela putorius\nn02443484 black-footed ferret, ferret, Mustela nigripes\nn02444819 otter\nn02445715 skunk, polecat, wood pussy\nn02447366 badger\nn02454379 armadillo\nn02457408 three-toed sloth, ai, Bradypus tridactylus\nn02480495 orangutan, orang, orangutang, Pongo pygmaeus\nn02480855 gorilla, Gorilla gorilla\nn02481823 chimpanzee, chimp, Pan troglodytes\nn02483362 gibbon, Hylobates lar\nn02483708 siamang, Hylobates syndactylus, Symphalangus syndactylus\nn02484975 guenon, guenon monkey\nn02486261 patas, hussar monkey, Erythrocebus patas\nn02486410 baboon\nn02487347 macaque\nn02488291 langur\nn02488702 colobus, colobus monkey\nn02489166 proboscis monkey, Nasalis larvatus\nn02490219 marmoset\nn02492035 capuchin, ringtail, Cebus capucinus\nn02492660 howler monkey, howler\nn02493509 titi, titi monkey\nn02493793 spider monkey, Ateles geoffroyi\nn02494079 squirrel monkey, Saimiri sciureus\nn02497673 Madagascar cat, ring-tailed lemur, Lemur catta\nn02500267 indri, indris, Indri indri, Indri brevicaudatus\nn02504013 Indian elephant, Elephas maximus\nn02504458 African elephant, Loxodonta africana\nn02509815 lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens\nn02510455 giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca\nn02514041 barracouta, snoek\nn02526121 eel\nn02536864 coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch\nn02606052 rock beauty, Holocanthus tricolor\nn02607072 anemone fish\nn02640242 sturgeon\nn02641379 gar, garfish, garpike, billfish, Lepisosteus osseus\nn02643566 lionfish\nn02655020 puffer, pufferfish, blowfish, globefish\nn02666196 abacus\nn02667093 abaya\nn02669723 academic gown, academic robe, judge's robe\nn02672831 accordion, piano accordion, squeeze box\nn02676566 acoustic guitar\nn02687172 aircraft carrier, carrier, flattop, attack aircraft carrier\nn02690373 airliner\nn02692877 airship, dirigible\nn02699494 altar\nn02701002 ambulance\nn02704792 amphibian, amphibious vehicle\nn02708093 analog clock\nn02727426 apiary, bee house\nn02730930 apron\nn02747177 ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin\nn02749479 assault rifle, assault gun\nn02769748 backpack, back pack, knapsack, packsack, rucksack, haversack\nn02776631 bakery, bakeshop, bakehouse\nn02777292 balance beam, beam\nn02782093 balloon\nn02783161 ballpoint, ballpoint pen, ballpen, Biro\nn02786058 Band Aid\nn02787622 banjo\nn02788148 bannister, banister, balustrade, balusters, handrail\nn02790996 barbell\nn02791124 barber chair\nn02791270 barbershop\nn02793495 barn\nn02794156 barometer\nn02795169 barrel, cask\nn02797295 barrow, garden cart, lawn cart, wheelbarrow\nn02799071 baseball\nn02802426 basketball\nn02804414 bassinet\nn02804610 bassoon\nn02807133 bathing cap, swimming cap\nn02808304 bath towel\nn02808440 bathtub, bathing tub, bath, tub\nn02814533 beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon\nn02814860 beacon, lighthouse, beacon light, pharos\nn02815834 beaker\nn02817516 bearskin, busby, shako\nn02823428 beer bottle\nn02823750 beer glass\nn02825657 bell cote, bell cot\nn02834397 bib\nn02835271 bicycle-built-for-two, tandem bicycle, tandem\nn02837789 bikini, two-piece\nn02840245 binder, ring-binder\nn02841315 binoculars, field glasses, opera glasses\nn02843684 birdhouse\nn02859443 boathouse\nn02860847 bobsled, bobsleigh, bob\nn02865351 bolo tie, bolo, bola tie, bola\nn02869837 bonnet, poke bonnet\nn02870880 bookcase\nn02871525 bookshop, bookstore, bookstall\nn02877765 bottlecap\nn02879718 bow\nn02883205 bow tie, bow-tie, bowtie\nn02892201 brass, memorial tablet, plaque\nn02892767 brassiere, bra, bandeau\nn02894605 breakwater, groin, groyne, mole, bulwark, seawall, jetty\nn02895154 breastplate, aegis, egis\nn02906734 broom\nn02909870 bucket, pail\nn02910353 buckle\nn02916936 bulletproof vest\nn02917067 bullet train, bullet\nn02927161 butcher shop, meat market\nn02930766 cab, hack, taxi, taxicab\nn02939185 caldron, cauldron\nn02948072 candle, taper, wax light\nn02950826 cannon\nn02951358 canoe\nn02951585 can opener, tin opener\nn02963159 cardigan\nn02965783 car mirror\nn02966193 carousel, carrousel, merry-go-round, roundabout, whirligig\nn02966687 carpenter's kit, tool kit\nn02971356 carton\nn02974003 car wheel\nn02977058 cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM\nn02978881 cassette\nn02979186 cassette player\nn02980441 castle\nn02981792 catamaran\nn02988304 CD player\nn02992211 cello, violoncello\nn02992529 cellular telephone, cellular phone, cellphone, cell, mobile phone\nn02999410 chain\nn03000134 chainlink fence\nn03000247 chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour\nn03000684 chain saw, chainsaw\nn03014705 chest\nn03016953 chiffonier, commode\nn03017168 chime, bell, gong\nn03018349 china cabinet, china closet\nn03026506 Christmas stocking\nn03028079 church, church building\nn03032252 cinema, movie theater, movie theatre, movie house, picture palace\nn03041632 cleaver, meat cleaver, chopper\nn03042490 cliff dwelling\nn03045698 cloak\nn03047690 clog, geta, patten, sabot\nn03062245 cocktail shaker\nn03063599 coffee mug\nn03063689 coffeepot\nn03065424 coil, spiral, volute, whorl, helix\nn03075370 combination lock\nn03085013 computer keyboard, keypad\nn03089624 confectionery, confectionary, candy store\nn03095699 container ship, containership, container vessel\nn03100240 convertible\nn03109150 corkscrew, bottle screw\nn03110669 cornet, horn, trumpet, trump\nn03124043 cowboy boot\nn03124170 cowboy hat, ten-gallon hat\nn03125729 cradle\nn03126707 crane\nn03127747 crash helmet\nn03127925 crate\nn03131574 crib, cot\nn03133878 Crock Pot\nn03134739 croquet ball\nn03141823 crutch\nn03146219 cuirass\nn03160309 dam, dike, dyke\nn03179701 desk\nn03180011 desktop computer\nn03187595 dial telephone, dial phone\nn03188531 diaper, nappy, napkin\nn03196217 digital clock\nn03197337 digital watch\nn03201208 dining table, board\nn03207743 dishrag, dishcloth\nn03207941 dishwasher, dish washer, dishwashing machine\nn03208938 disk brake, disc brake\nn03216828 dock, dockage, docking facility\nn03218198 dogsled, dog sled, dog sleigh\nn03220513 dome\nn03223299 doormat, welcome mat\nn03240683 drilling platform, offshore rig\nn03249569 drum, membranophone, tympan\nn03250847 drumstick\nn03255030 dumbbell\nn03259280 Dutch oven\nn03271574 electric fan, blower\nn03272010 electric guitar\nn03272562 electric locomotive\nn03290653 entertainment center\nn03291819 envelope\nn03297495 espresso maker\nn03314780 face powder\nn03325584 feather boa, boa\nn03337140 file, file cabinet, filing cabinet\nn03344393 fireboat\nn03345487 fire engine, fire truck\nn03347037 fire screen, fireguard\nn03355925 flagpole, flagstaff\nn03372029 flute, transverse flute\nn03376595 folding chair\nn03379051 football helmet\nn03384352 forklift\nn03388043 fountain\nn03388183 fountain pen\nn03388549 four-poster\nn03393912 freight car\nn03394916 French horn, horn\nn03400231 frying pan, frypan, skillet\nn03404251 fur coat\nn03417042 garbage truck, dustcart\nn03424325 gasmask, respirator, gas helmet\nn03425413 gas pump, gasoline pump, petrol pump, island dispenser\nn03443371 goblet\nn03444034 go-kart\nn03445777 golf ball\nn03445924 golfcart, golf cart\nn03447447 gondola\nn03447721 gong, tam-tam\nn03450230 gown\nn03452741 grand piano, grand\nn03457902 greenhouse, nursery, glasshouse\nn03459775 grille, radiator grille\nn03461385 grocery store, grocery, food market, market\nn03467068 guillotine\nn03476684 hair slide\nn03476991 hair spray\nn03478589 half track\nn03481172 hammer\nn03482405 hamper\nn03483316 hand blower, blow dryer, blow drier, hair dryer, hair drier\nn03485407 hand-held computer, hand-held microcomputer\nn03485794 handkerchief, hankie, hanky, hankey\nn03492542 hard disc, hard disk, fixed disk\nn03494278 harmonica, mouth organ, harp, mouth harp\nn03495258 harp\nn03496892 harvester, reaper\nn03498962 hatchet\nn03527444 holster\nn03529860 home theater, home theatre\nn03530642 honeycomb\nn03532672 hook, claw\nn03534580 hoopskirt, crinoline\nn03535780 horizontal bar, high bar\nn03538406 horse cart, horse-cart\nn03544143 hourglass\nn03584254 iPod\nn03584829 iron, smoothing iron\nn03590841 jack-o'-lantern\nn03594734 jean, blue jean, denim\nn03594945 jeep, landrover\nn03595614 jersey, T-shirt, tee shirt\nn03598930 jigsaw puzzle\nn03599486 jinrikisha, ricksha, rickshaw\nn03602883 joystick\nn03617480 kimono\nn03623198 knee pad\nn03627232 knot\nn03630383 lab coat, laboratory coat\nn03633091 ladle\nn03637318 lampshade, lamp shade\nn03642806 laptop, laptop computer\nn03649909 lawn mower, mower\nn03657121 lens cap, lens cover\nn03658185 letter opener, paper knife, paperknife\nn03661043 library\nn03662601 lifeboat\nn03666591 lighter, light, igniter, ignitor\nn03670208 limousine, limo\nn03673027 liner, ocean liner\nn03676483 lipstick, lip rouge\nn03680355 Loafer\nn03690938 lotion\nn03691459 loudspeaker, speaker, speaker unit, loudspeaker system, speaker system\nn03692522 loupe, jeweler's loupe\nn03697007 lumbermill, sawmill\nn03706229 magnetic compass\nn03709823 mailbag, postbag\nn03710193 mailbox, letter box\nn03710637 maillot\nn03710721 maillot, tank suit\nn03717622 manhole cover\nn03720891 maraca\nn03721384 marimba, xylophone\nn03724870 mask\nn03729826 matchstick\nn03733131 maypole\nn03733281 maze, labyrinth\nn03733805 measuring cup\nn03742115 medicine chest, medicine cabinet\nn03743016 megalith, megalithic structure\nn03759954 microphone, mike\nn03761084 microwave, microwave oven\nn03763968 military uniform\nn03764736 milk can\nn03769881 minibus\nn03770439 miniskirt, mini\nn03770679 minivan\nn03773504 missile\nn03775071 mitten\nn03775546 mixing bowl\nn03776460 mobile home, manufactured home\nn03777568 Model T\nn03777754 modem\nn03781244 monastery\nn03782006 monitor\nn03785016 moped\nn03786901 mortar\nn03787032 mortarboard\nn03788195 mosque\nn03788365 mosquito net\nn03791053 motor scooter, scooter\nn03792782 mountain bike, all-terrain bike, off-roader\nn03792972 mountain tent\nn03793489 mouse, computer mouse\nn03794056 mousetrap\nn03796401 moving van\nn03803284 muzzle\nn03804744 nail\nn03814639 neck brace\nn03814906 necklace\nn03825788 nipple\nn03832673 notebook, notebook computer\nn03837869 obelisk\nn03838899 oboe, hautboy, hautbois\nn03840681 ocarina, sweet potato\nn03841143 odometer, hodometer, mileometer, milometer\nn03843555 oil filter\nn03854065 organ, pipe organ\nn03857828 oscilloscope, scope, cathode-ray oscilloscope, CRO\nn03866082 overskirt\nn03868242 oxcart\nn03868863 oxygen mask\nn03871628 packet\nn03873416 paddle, boat paddle\nn03874293 paddlewheel, paddle wheel\nn03874599 padlock\nn03876231 paintbrush\nn03877472 pajama, pyjama, pj's, jammies\nn03877845 palace\nn03884397 panpipe, pandean pipe, syrinx\nn03887697 paper towel\nn03888257 parachute, chute\nn03888605 parallel bars, bars\nn03891251 park bench\nn03891332 parking meter\nn03895866 passenger car, coach, carriage\nn03899768 patio, terrace\nn03902125 pay-phone, pay-station\nn03903868 pedestal, plinth, footstall\nn03908618 pencil box, pencil case\nn03908714 pencil sharpener\nn03916031 perfume, essence\nn03920288 Petri dish\nn03924679 photocopier\nn03929660 pick, plectrum, plectron\nn03929855 pickelhaube\nn03930313 picket fence, paling\nn03930630 pickup, pickup truck\nn03933933 pier\nn03935335 piggy bank, penny bank\nn03937543 pill bottle\nn03938244 pillow\nn03942813 ping-pong ball\nn03944341 pinwheel\nn03947888 pirate, pirate ship\nn03950228 pitcher, ewer\nn03954731 plane, carpenter's plane, woodworking plane\nn03956157 planetarium\nn03958227 plastic bag\nn03961711 plate rack\nn03967562 plow, plough\nn03970156 plunger, plumber's helper\nn03976467 Polaroid camera, Polaroid Land camera\nn03976657 pole\nn03977966 police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria\nn03980874 poncho\nn03982430 pool table, billiard table, snooker table\nn03983396 pop bottle, soda bottle\nn03991062 pot, flowerpot\nn03992509 potter's wheel\nn03995372 power drill\nn03998194 prayer rug, prayer mat\nn04004767 printer\nn04005630 prison, prison house\nn04008634 projectile, missile\nn04009552 projector\nn04019541 puck, hockey puck\nn04023962 punching bag, punch bag, punching ball, punchball\nn04026417 purse\nn04033901 quill, quill pen\nn04033995 quilt, comforter, comfort, puff\nn04037443 racer, race car, racing car\nn04039381 racket, racquet\nn04040759 radiator\nn04041544 radio, wireless\nn04044716 radio telescope, radio reflector\nn04049303 rain barrel\nn04065272 recreational vehicle, RV, R.V.\nn04067472 reel\nn04069434 reflex camera\nn04070727 refrigerator, icebox\nn04074963 remote control, remote\nn04081281 restaurant, eating house, eating place, eatery\nn04086273 revolver, six-gun, six-shooter\nn04090263 rifle\nn04099969 rocking chair, rocker\nn04111531 rotisserie\nn04116512 rubber eraser, rubber, pencil eraser\nn04118538 rugby ball\nn04118776 rule, ruler\nn04120489 running shoe\nn04125021 safe\nn04127249 safety pin\nn04131690 saltshaker, salt shaker\nn04133789 sandal\nn04136333 sarong\nn04141076 sax, saxophone\nn04141327 scabbard\nn04141975 scale, weighing machine\nn04146614 school bus\nn04147183 schooner\nn04149813 scoreboard\nn04152593 screen, CRT screen\nn04153751 screw\nn04154565 screwdriver\nn04162706 seat belt, seatbelt\nn04179913 sewing machine\nn04192698 shield, buckler\nn04200800 shoe shop, shoe-shop, shoe store\nn04201297 shoji\nn04204238 shopping basket\nn04204347 shopping cart\nn04208210 shovel\nn04209133 shower cap\nn04209239 shower curtain\nn04228054 ski\nn04229816 ski mask\nn04235860 sleeping bag\nn04238763 slide rule, slipstick\nn04239074 sliding door\nn04243546 slot, one-armed bandit\nn04251144 snorkel\nn04252077 snowmobile\nn04252225 snowplow, snowplough\nn04254120 soap dispenser\nn04254680 soccer ball\nn04254777 sock\nn04258138 solar dish, solar collector, solar furnace\nn04259630 sombrero\nn04263257 soup bowl\nn04264628 space bar\nn04265275 space heater\nn04266014 space shuttle\nn04270147 spatula\nn04273569 speedboat\nn04275548 spider web, spider's web\nn04277352 spindle\nn04285008 sports car, sport car\nn04286575 spotlight, spot\nn04296562 stage\nn04310018 steam locomotive\nn04311004 steel arch bridge\nn04311174 steel drum\nn04317175 stethoscope\nn04325704 stole\nn04326547 stone wall\nn04328186 stopwatch, stop watch\nn04330267 stove\nn04332243 strainer\nn04335435 streetcar, tram, tramcar, trolley, trolley car\nn04336792 stretcher\nn04344873 studio couch, day bed\nn04346328 stupa, tope\nn04347754 submarine, pigboat, sub, U-boat\nn04350905 suit, suit of clothes\nn04355338 sundial\nn04355933 sunglass\nn04356056 sunglasses, dark glasses, shades\nn04357314 sunscreen, sunblock, sun blocker\nn04366367 suspension bridge\nn04367480 swab, swob, mop\nn04370456 sweatshirt\nn04371430 swimming trunks, bathing trunks\nn04371774 swing\nn04372370 switch, electric switch, electrical switch\nn04376876 syringe\nn04380533 table lamp\nn04389033 tank, army tank, armored combat vehicle, armoured combat vehicle\nn04392985 tape player\nn04398044 teapot\nn04399382 teddy, teddy bear\nn04404412 television, television system\nn04409515 tennis ball\nn04417672 thatch, thatched roof\nn04418357 theater curtain, theatre curtain\nn04423845 thimble\nn04428191 thresher, thrasher, threshing machine\nn04429376 throne\nn04435653 tile roof\nn04442312 toaster\nn04443257 tobacco shop, tobacconist shop, tobacconist\nn04447861 toilet seat\nn04456115 torch\nn04458633 totem pole\nn04461696 tow truck, tow car, wrecker\nn04462240 toyshop\nn04465501 tractor\nn04467665 trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi\nn04476259 tray\nn04479046 trench coat\nn04482393 tricycle, trike, velocipede\nn04483307 trimaran\nn04485082 tripod\nn04486054 triumphal arch\nn04487081 trolleybus, trolley coach, trackless trolley\nn04487394 trombone\nn04493381 tub, vat\nn04501370 turnstile\nn04505470 typewriter keyboard\nn04507155 umbrella\nn04509417 unicycle, monocycle\nn04515003 upright, upright piano\nn04517823 vacuum, vacuum cleaner\nn04522168 vase\nn04523525 vault\nn04525038 velvet\nn04525305 vending machine\nn04532106 vestment\nn04532670 viaduct\nn04536866 violin, fiddle\nn04540053 volleyball\nn04542943 waffle iron\nn04548280 wall clock\nn04548362 wallet, billfold, notecase, pocketbook\nn04550184 wardrobe, closet, press\nn04552348 warplane, military plane\nn04553703 washbasin, handbasin, washbowl, lavabo, wash-hand basin\nn04554684 washer, automatic washer, washing machine\nn04557648 water bottle\nn04560804 water jug\nn04562935 water tower\nn04579145 whiskey jug\nn04579432 whistle\nn04584207 wig\nn04589890 window screen\nn04590129 window shade\nn04591157 Windsor tie\nn04591713 wine bottle\nn04592741 wing\nn04596742 wok\nn04597913 wooden spoon\nn04599235 wool, woolen, woollen\nn04604644 worm fence, snake fence, snake-rail fence, Virginia fence\nn04606251 wreck\nn04612504 yawl\nn04613696 yurt\nn06359193 web site, website, internet site, site\nn06596364 comic book\nn06785654 crossword puzzle, crossword\nn06794110 street sign\nn06874185 traffic light, traffic signal, stoplight\nn07248320 book jacket, dust cover, dust jacket, dust wrapper\nn07565083 menu\nn07579787 plate\nn07583066 guacamole\nn07584110 consomme\nn07590611 hot pot, hotpot\nn07613480 trifle\nn07614500 ice cream, icecream\nn07615774 ice lolly, lolly, lollipop, popsicle\nn07684084 French loaf\nn07693725 bagel, beigel\nn07695742 pretzel\nn07697313 cheeseburger\nn07697537 hotdog, hot dog, red hot\nn07711569 mashed potato\nn07714571 head cabbage\nn07714990 broccoli\nn07715103 cauliflower\nn07716358 zucchini, courgette\nn07716906 spaghetti squash\nn07717410 acorn squash\nn07717556 butternut squash\nn07718472 cucumber, cuke\nn07718747 artichoke, globe artichoke\nn07720875 bell pepper\nn07730033 cardoon\nn07734744 mushroom\nn07742313 Granny Smith\nn07745940 strawberry\nn07747607 orange\nn07749582 lemon\nn07753113 fig\nn07753275 pineapple, ananas\nn07753592 banana\nn07754684 jackfruit, jak, jack\nn07760859 custard apple\nn07768694 pomegranate\nn07802026 hay\nn07831146 carbonara\nn07836838 chocolate sauce, chocolate syrup\nn07860988 dough\nn07871810 meat loaf, meatloaf\nn07873807 pizza, pizza pie\nn07875152 potpie\nn07880968 burrito\nn07892512 red wine\nn07920052 espresso\nn07930864 cup\nn07932039 eggnog\nn09193705 alp\nn09229709 bubble\nn09246464 cliff, drop, drop-off\nn09256479 coral reef\nn09288635 geyser\nn09332890 lakeside, lakeshore\nn09399592 promontory, headland, head, foreland\nn09421951 sandbar, sand bar\nn09428293 seashore, coast, seacoast, sea-coast\nn09468604 valley, vale\nn09472597 volcano\nn09835506 ballplayer, baseball player\nn10148035 groom, bridegroom\nn10565667 scuba diver\nn11879895 rapeseed\nn11939491 daisy\nn12057211 yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum\nn12144580 corn\nn12267677 acorn\nn12620546 hip, rose hip, rosehip\nn12768682 buckeye, horse chestnut, conker\nn12985857 coral fungus\nn12998815 agaric\nn13037406 gyromitra\nn13040303 stinkhorn, carrion fungus\nn13044778 earthstar\nn13052670 hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa\nn13054560 bolete\nn13133613 ear, spike, capitulum\nn15075141 toilet tissue, toilet paper, bathroom tissue\n" }, { "path": "li_attack.py", "content": "## li_attack.py -- attack a network optimizing for l_infinity distance\n##\n## Copyright (C) IBM Corp, 2017-2018\n## Copyright (C) 2016, Nicholas Carlini .\n##\n## This program is licenced under the BSD 2-Clause licence,\n## contained in the LICENCE file in this directory.\n\nimport sys\nimport tensorflow as tf\nimport numpy as np\n\nDECREASE_FACTOR = 0.9 # 01, rate at which we increase constant, smaller better\n\nclass CarliniLi:\n def __init__(self, sess, model,\n targeted = TARGETED, learning_rate = LEARNING_RATE,\n max_iterations = MAX_ITERATIONS, abort_early = ABORT_EARLY,\n initial_const = INITIAL_CONST, largest_const = LARGEST_CONST,\n reduce_const = REDUCE_CONST, decrease_factor = DECREASE_FACTOR,\n const_factor = CONST_FACTOR):\n \"\"\"\n The L_infinity optimized attack. \n\n Returns adversarial examples for the supplied model.\n\n targeted: True if we should perform a targetted attack, False otherwise.\n learning_rate: The learning rate for the attack algorithm. Smaller values\n produce better results but are slower to converge.\n max_iterations: The maximum number of iterations. Larger values are more\n accurate; setting too small will require a large learning rate and will\n produce poor results.\n abort_early: If true, allows early aborts if gradient descent gets stuck.\n initial_const: The initial tradeoff-constant to use to tune the relative\n importance of distance and confidence. Should be set to a very small\n value (but positive).\n largest_const: The largest constant to use until we report failure. Should\n be set to a very large value.\n reduce_const: If true, after each successful attack, make const smaller.\n decrease_factor: Rate at which we should decrease tau, less than one.\n Larger produces better quality results.\n const_factor: The rate at which we should increase the constant, when the\n previous constant failed. Should be greater than one, smaller is better.\n \"\"\"\n self.model = model\n self.sess = sess\n\n self.TARGETED = targeted\n self.LEARNING_RATE = learning_rate\n self.MAX_ITERATIONS = max_iterations\n self.ABORT_EARLY = abort_early\n self.INITIAL_CONST = initial_const\n self.LARGEST_CONST = largest_const\n self.DECREASE_FACTOR = decrease_factor\n self.REDUCE_CONST = reduce_const\n self.const_factor = const_factor\n\n self.grad = self.gradient_descent(sess, model)\n\n def gradient_descent(self, sess, model):\n def compare(x,y):\n if self.TARGETED:\n return x == y\n else:\n return x != y\n shape = (1,model.image_size,model.image_size,model.num_channels)\n \n # the variable to optimize over\n modifier = tf.Variable(np.zeros(shape,dtype=np.float32))\n\n tau = tf.placeholder(tf.float32, [])\n simg = tf.placeholder(tf.float32, shape)\n timg = tf.placeholder(tf.float32, shape)\n tlab = tf.placeholder(tf.float32, (1,model.num_labels))\n const = tf.placeholder(tf.float32, [])\n \n newimg = (tf.tanh(modifier + simg)/2)\n \n output = model.predict(newimg)\n orig_output = model.predict(tf.tanh(timg)/2)\n \n real = tf.reduce_sum((tlab)*output)\n other = tf.reduce_max((1-tlab)*output - (tlab*10000))\n \n if self.TARGETED:\n # if targetted, optimize for making the other class most likely\n loss1 = tf.maximum(0.0,other-real)\n else:\n # if untargeted, optimize for making this class least likely.\n loss1 = tf.maximum(0.0,real-other)\n\n # sum up the losses\n loss2 = tf.reduce_sum(tf.maximum(0.0,tf.abs(newimg-tf.tanh(timg)/2)-tau))\n loss = const*loss1+loss2\n \n # setup the adam optimizer and keep track of variables we're creating\n start_vars = set(x.name for x in tf.global_variables())\n optimizer = tf.train.AdamOptimizer(self.LEARNING_RATE)\n train = optimizer.minimize(loss, var_list=[modifier])\n\n end_vars = tf.global_variables()\n new_vars = [x for x in end_vars if x.name not in start_vars]\n init = tf.variables_initializer(var_list=[modifier]+new_vars)\n \n def doit(oimgs, labs, starts, tt, CONST):\n # convert to tanh-space\n imgs = np.arctanh(np.array(oimgs)*1.999999)\n starts = np.arctanh(np.array(starts)*1.999999)\n \n # initialize the variables\n sess.run(init)\n while CONST < self.LARGEST_CONST:\n # try solving for each value of the constant\n print('try const', CONST)\n for step in range(self.MAX_ITERATIONS):\n feed_dict={timg: imgs, \n tlab:labs, \n tau: tt,\n simg: starts,\n const: CONST}\n if step%(self.MAX_ITERATIONS//10) == 0:\n print(step,sess.run((loss,loss1,loss2),feed_dict=feed_dict))\n\n # perform the update step\n _, works = sess.run([train, loss], feed_dict=feed_dict)\n \n # it worked\n if works < .0001*CONST and (self.ABORT_EARLY or step == CONST-1):\n get = sess.run(output, feed_dict=feed_dict)\n works = compare(np.argmax(get), np.argmax(labs))\n if works:\n scores, origscores, nimg = sess.run((output,orig_output,newimg),feed_dict=feed_dict)\n l2s=np.square(nimg-np.tanh(imgs)/2).sum(axis=(1,2,3))\n \n return scores, origscores, nimg, CONST\n\n # we didn't succeed, increase constant and try again\n CONST *= self.const_factor\n \n return doit\n \n def attack(self, imgs, targets):\n \"\"\"\n Perform the L_0 attack on the given images for the given targets.\n\n If self.targeted is true, then the targets represents the target labels.\n If self.targeted is false, then targets are the original class labels.\n \"\"\"\n r = []\n for img,target in zip(imgs, targets):\n r.extend(self.attack_single(img, target))\n return np.array(r)\n\n def attack_single(self, img, target):\n \"\"\"\n Run the attack on a single image and label\n \"\"\"\n\n # the previous image\n prev = np.copy(img).reshape((1,self.model.image_size,self.model.image_size,self.model.num_channels))\n tau = 1.0\n const = self.INITIAL_CONST\n \n while tau > 1./256:\n # try to solve given this tau value\n res = self.grad([np.copy(img)], [target], np.copy(prev), tau, const)\n if res == None:\n # the attack failed, we return this as our final answer\n return prev\n \n scores, origscores, nimg, const = res\n if self.REDUCE_CONST: const /= 2\n\n # the attack succeeded, reduce tau and try again\n \n actualtau = np.max(np.abs(nimg-img))\n \n if actualtau < tau:\n tau = actualtau\n \n print(\"Tau\",tau)\n\n prev = nimg\n tau *= self.DECREASE_FACTOR\n return prev\n" }, { "path": "mnist_blackbox.py", "content": "## Copyright (C) IBM Corp, 2017-2018\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\n\nimport os\nimport time\nimport numpy as np\nfrom six.moves import xrange\n\nimport keras\nfrom keras import backend\nfrom keras.utils.np_utils import to_categorical\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Flatten, Activation, Dropout\n\nimport tensorflow as tf\nfrom tensorflow.python.platform import app\nfrom tensorflow.python.platform import flags\n\nfrom cleverhans.utils_keras import cnn_model\nfrom cleverhans.utils_mnist import data_mnist\nfrom cleverhans.utils_tf import model_train, model_eval, batch_eval, tf_model_load\nfrom cleverhans.attacks import CarliniWagnerL2\nfrom cleverhans.attacks import FastGradientMethod\nfrom cleverhans.attacks_tf import jacobian_graph, jacobian_augmentation\nfrom cleverhans.utils_keras import KerasModelWrapper\n\nfrom setup_mnist import MNISTModel\n\nFLAGS = flags.FLAGS\n\n\ndef setup_tutorial():\n \"\"\"\n Helper function to check correct configuration of tf and keras for tutorial\n :return: True if setup checks completed\n \"\"\"\n\n # Set TF random seed to improve reproducibility\n tf.set_random_seed(1234)\n\n if not hasattr(backend, \"tf\"):\n raise RuntimeError(\"This tutorial requires keras to be configured\"\n \" to use the TensorFlow backend.\")\n\n # Image dimensions ordering should follow the Theano convention\n if keras.backend.image_dim_ordering() != 'tf':\n keras.backend.set_image_dim_ordering('tf')\n print(\"INFO: '~/.keras/keras.json' sets 'image_dim_ordering' \"\n \"to 'th', temporarily setting to 'tf'\")\n\n return True\n\n\ndef prep_bbox(sess, x, y, X_train, Y_train, X_test, Y_test,\n nb_epochs, batch_size, learning_rate):\n \"\"\"\n Define and train a model that simulates the \"remote\"\n black-box oracle described in the original paper.\n :param sess: the TF session\n :param x: the input placeholder for MNIST\n :param y: the ouput placeholder for MNIST\n :param X_train: the training data for the oracle\n :param Y_train: the training labels for the oracle\n :param X_test: the testing data for the oracle\n :param Y_test: the testing labels for the oracle\n :param nb_epochs: number of epochs to train model\n :param batch_size: size of training batches\n :param learning_rate: learning rate for training\n :return:\n \"\"\"\n\n # Define TF model graph (for the black-box model)\n model = MNISTModel(use_log = True).model\n predictions = model(x)\n print(\"Defined TensorFlow model graph.\")\n\n # Train an MNIST model\n if FLAGS.load_pretrain:\n tf_model_load(sess)\n else:\n train_params = {\n 'nb_epochs': nb_epochs,\n 'batch_size': batch_size,\n 'learning_rate': learning_rate\n }\n model_train(sess, x, y, predictions, X_train, Y_train, verbose=False, save=True,\n args=train_params)\n\n # Print out the accuracy on legitimate data\n eval_params = {'batch_size': batch_size}\n accuracy = model_eval(sess, x, y, predictions, X_test, Y_test,\n args=eval_params)\n print('Test accuracy of black-box on legitimate test '\n 'examples: ' + str(accuracy))\n\n return model, predictions, accuracy\n\n\ndef substitute_model(img_rows=28, img_cols=28, nb_classes=10):\n \"\"\"\n Defines the model architecture to be used by the substitute\n :param img_rows: number of rows in input\n :param img_cols: number of columns in input\n :param nb_classes: number of classes in output\n :return: keras model\n \"\"\"\n model = Sequential()\n\n # Find out the input shape ordering\n if keras.backend.image_dim_ordering() == 'th':\n input_shape = (1, img_rows, img_cols)\n else:\n input_shape = (img_rows, img_cols, 1)\n\n # Define a fully connected model (it's different than the black-box)\n layers = [Flatten(input_shape=input_shape),\n Dense(200),\n Activation('relu'),\n Dropout(0.5),\n Dense(200),\n Activation('relu'),\n Dropout(0.5),\n Dense(nb_classes),\n Activation('softmax')]\n\n for layer in layers:\n model.add(layer)\n\n return model\n\n\ndef train_sub(sess, x, y, bbox_preds, X_sub, Y_sub, nb_classes,\n nb_epochs_s, batch_size, learning_rate, data_aug, lmbda):\n \"\"\"\n This function creates the substitute by alternatively\n augmenting the training data and training the substitute.\n :param sess: TF session\n :param x: input TF placeholder\n :param y: output TF placeholder\n :param bbox_preds: output of black-box model predictions\n :param X_sub: initial substitute training data\n :param Y_sub: initial substitute training labels\n :param nb_classes: number of output classes\n :param nb_epochs_s: number of epochs to train substitute model\n :param batch_size: size of training batches\n :param learning_rate: learning rate for training\n :param data_aug: number of times substitute training data is augmented\n :param lmbda: lambda from arxiv.org/abs/1602.02697\n :return:\n \"\"\"\n # Define TF model graph (for the black-box model)\n # model_sub = substitute_model()\n model_sub = MNISTModel(use_log = True).model\n preds_sub = model_sub(x)\n print(\"Defined TensorFlow model graph for the substitute.\")\n\n # Define the Jacobian symbolically using TensorFlow\n grads = jacobian_graph(preds_sub, x, nb_classes)\n\n # Train the substitute and augment dataset alternatively\n for rho in xrange(data_aug):\n print(\"Substitute training epoch #\" + str(rho))\n train_params = {\n 'nb_epochs': nb_epochs_s,\n 'batch_size': batch_size,\n 'learning_rate': learning_rate\n }\n model_train(sess, x, y, preds_sub, X_sub, to_categorical(Y_sub),\n init_all=False, verbose=False, args=train_params)\n\n # If we are not at last substitute training iteration, augment dataset\n if rho < data_aug - 1:\n if FLAGS.cached_aug:\n augs = np.load('sub_saved/mnist-aug-{}.npz'.format(rho))\n X_sub = augs['X_sub']\n Y_sub = augs['Y_sub']\n else:\n print(\"Augmenting substitute training data.\")\n # Perform the Jacobian augmentation\n X_sub = jacobian_augmentation(sess, x, X_sub, Y_sub, grads, lmbda)\n\n print(\"Labeling substitute training data.\")\n # Label the newly generated synthetic points using the black-box\n Y_sub = np.hstack([Y_sub, Y_sub])\n X_sub_prev = X_sub[int(len(X_sub)/2):]\n eval_params = {'batch_size': batch_size}\n bbox_val = batch_eval(sess, [x], [bbox_preds], [X_sub_prev],\n args=eval_params)[0]\n # Note here that we take the argmax because the adversary\n # only has access to the label (not the probabilities) output\n # by the black-box model\n Y_sub[int(len(X_sub)/2):] = np.argmax(bbox_val, axis=1)\n # cache the augmentation\n if not FLAGS.cached_aug:\n np.savez('sub_saved/mnist-aug-{}.npz'.format(rho), X_sub = X_sub, Y_sub = Y_sub)\n\n return model_sub, preds_sub\n\n\ndef mnist_blackbox(train_start=0, train_end=60000, test_start=0,\n test_end=10000, nb_classes=10, batch_size=128,\n learning_rate=0.001, nb_epochs=10, holdout=150, data_aug=6,\n nb_epochs_s=10, lmbda=0.1, attack=\"fgsm\", targeted=False):\n \"\"\"\n MNIST tutorial for the black-box attack from arxiv.org/abs/1602.02697\n :param train_start: index of first training set example\n :param train_end: index of last training set example\n :param test_start: index of first test set example\n :param test_end: index of last test set example\n :return: a dictionary with:\n * black-box model accuracy on test set\n * substitute model accuracy on test set\n * black-box model accuracy on adversarial examples transferred\n from the substitute model\n \"\"\"\n keras.layers.core.K.set_learning_phase(0)\n\n # Dictionary used to keep track and return key accuracies\n accuracies = {}\n\n # Perform tutorial setup\n assert setup_tutorial()\n\n # Create TF session and set as Keras backend session\n gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.45)\n sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))\n keras.backend.set_session(sess)\n\n # Get MNIST data\n X_train, Y_train, X_test, Y_test = data_mnist(train_start=train_start,\n train_end=train_end,\n test_start=test_start,\n test_end=test_end)\n\n # Initialize substitute training set reserved for adversary\n X_sub = X_test[:holdout]\n Y_sub = np.argmax(Y_test[:holdout], axis=1)\n\n # Redefine test set as remaining samples unavailable to adversaries\n X_test = X_test[holdout:]\n Y_test = Y_test[holdout:]\n\n X_test = X_test[:FLAGS.n_attack]\n Y_test = Y_test[:FLAGS.n_attack]\n\n # Define input and output TF placeholders\n x = tf.placeholder(tf.float32, shape=(None, 28, 28, 1))\n y = tf.placeholder(tf.float32, shape=(None, 10))\n\n # Simulate the black-box model locally\n # You could replace this by a remote labeling API for instance\n print(\"Preparing the black-box model.\")\n prep_bbox_out = prep_bbox(sess, x, y, X_train, Y_train, X_test, Y_test,\n nb_epochs, batch_size, learning_rate)\n model, bbox_preds, accuracies['bbox'] = prep_bbox_out\n\n # Train substitute using method from https://arxiv.org/abs/1602.02697\n time_start = time.time()\n print(\"Training the substitute model.\")\n train_sub_out = train_sub(sess, x, y, bbox_preds, X_sub, Y_sub,\n nb_classes, nb_epochs_s, batch_size,\n learning_rate, data_aug, lmbda)\n model_sub, preds_sub = train_sub_out\n time_end = time.time()\n print(\"Substitue model training time:\", time_end - time_start)\n\n # Evaluate the substitute model on clean test examples\n eval_params = {'batch_size': batch_size}\n acc = model_eval(sess, x, y, preds_sub, X_test, Y_test, args=eval_params)\n accuracies['sub'] = acc\n print('substitution model accuracy:', acc)\n\n # Find the correctly predicted labels\n original_predict = batch_eval(sess, [x], [bbox_preds], [X_test],\n args=eval_params)[0]\n original_class = np.argmax(original_predict, axis = 1)\n true_class = np.argmax(Y_test, axis = 1)\n mask = true_class == original_class\n print(np.sum(mask), \"out of\", mask.size, \"are correct labeled,\", len(X_test[mask])) \n\n # Initialize the Fast Gradient Sign Method (FGSM) attack object.\n wrap = KerasModelWrapper(model_sub)\n\n\n # Craft adversarial examples using the substitute\n eval_params = {'batch_size': batch_size}\n\n if attack == \"fgsm\":\n attacker_params = {'eps': 0.4, 'ord': np.inf, 'clip_min': 0., 'clip_max': 1.}\n fgsm = FastGradientMethod(wrap, sess=sess)\n x_adv_sub = fgsm.generate(x, **attacker_params)\n attacker = fgsm\n adv_inputs = X_test\n ori_labels = Y_test\n print(\"Running FGSM attack...\")\n else:\n print(\"Running Carlini and Wagner\\'s L2 attack...\")\n yname = \"y\"\n adv_ys = None\n # wrap = KerasModelWrapper(model)\n cwl2 = CarliniWagnerL2(wrap, back='tf', sess=sess)\n attacker_params = {'binary_search_steps': 9,\n 'max_iterations': 2000,\n 'abort_early': True,\n 'learning_rate': 0.01,\n 'batch_size': 1,\n 'initial_const': 0.01,\n 'confidence': 20}\n # generate targeted labels, 9 for each test example\n if targeted:\n adv_ys = []\n targeted_class = []\n for i in range(0, X_test.shape[0]):\n for j in range(0,10):\n # skip the original image label\n if j == np.argmax(Y_test[i]):\n continue\n adv_ys.append(np.eye(10)[j])\n targeted_class.append(j)\n attacker_params['y_target'] = np.array(adv_ys, dtype=np.float32)\n # duplicate the inputs by 9 times\n adv_inputs = np.array([[instance] * 9 for instance in X_test],\n dtype=np.float32)\n adv_inputs = adv_inputs.reshape((X_test.shape[0] * 9, 28, 28, 1))\n # also update the mask\n mask = np.repeat(mask, 9)\n ori_labels = np.repeat(Y_test, 9, axis=0)\n else:\n adv_inputs = X_test\n ori_labels = Y_test\n attacker = cwl2\n\n if attack == \"fgsm\":\n # Evaluate the accuracy of the \"black-box\" model on adversarial examples\n accuracy = model_eval(sess, x, y, model(x_adv_sub), adv_inputs, ori_labels,\n args=eval_params)\n print('Test accuracy of oracle on adversarial examples generated '\n 'using the substitute: ' + str(accuracy))\n accuracies['bbox_on_sub_adv_ex'] = accuracy\n\n time_start = time.time()\n # Evaluate the accuracy of the \"black-box\" model on adversarial examples\n x_adv_sub_np = attacker.generate_np(adv_inputs, **attacker_params)\n accuracy = model_eval(sess, x, y, bbox_preds, x_adv_sub_np, ori_labels,\n args=eval_params)\n print('Test accuracy of oracle on adversarial examples generated '\n 'using the substitute (NP): ' + str(accuracy))\n accuracies['bbox_on_sub_adv_ex'] = accuracy\n time_end = time.time()\n print('Attack time:', time_end - time_start)\n\n # Evaluate the targeted attack\n bbox_adv_predict = batch_eval(sess, [x], [bbox_preds], [x_adv_sub_np],\n args=eval_params)[0]\n bbox_adv_class = np.argmax(bbox_adv_predict, axis = 1)\n true_class = np.argmax(ori_labels, axis = 1)\n untargeted_success = np.mean(bbox_adv_class != true_class)\n print('Untargeted attack success rate:', untargeted_success)\n accuracies['untargeted_success'] = untargeted_success\n if targeted:\n targeted_success = np.mean(bbox_adv_class == targeted_class)\n print('Targeted attack success rate:', targeted_success)\n accuracies['targeted_success'] = targeted_success\n\n if attack == \"cwl2\":\n # Compute the L2 pertubations of generated adversarial examples\n percent_perturbed = np.sum((x_adv_sub_np - adv_inputs)**2, axis=(1, 2, 3))**.5\n # print(percent_perturbed)\n # print('Avg. L_2 norm of perturbations {0:.4f}'.format(np.mean(percent_perturbed)))\n # when computing the mean, removing the failure attacks first\n print('Avg. L_2 norm of perturbations {0:.4f}'.format(np.mean(percent_perturbed[percent_perturbed > 1e-8])))\n\n # Evaluate the accuracy of the \"black-box\" model on adversarial examples\n accuracy = model_eval(sess, x, y, bbox_preds, adv_inputs[mask], ori_labels[mask],\n args=eval_params)\n print('Test accuracy of excluding originally incorrect labels (should be 1.0): ' + str(accuracy))\n accuracies['bbox_on_sub_adv_ex_exc_ori'] = accuracy\n\n if attack == \"fgsm\":\n # Evaluate the accuracy of the \"black-box\" model on adversarial examples (excluding correct)\n accuracy = model_eval(sess, x, y, model(x_adv_sub), adv_inputs[mask], ori_labels[mask],\n args=eval_params)\n print('Test accuracy of oracle on adversarial examples generated '\n 'using the substitute (excluding originally incorrect labels): ' + str(accuracy))\n accuracies['bbox_on_sub_adv_ex_exc'] = accuracy\n\n # Evaluate the accuracy of the \"black-box\" model on adversarial examples (excluding correct)\n x_adv_sub_mask_np = x_adv_sub_np[mask]\n accuracy = model_eval(sess, x, y, bbox_preds, x_adv_sub_mask_np, ori_labels[mask],\n args=eval_params)\n print('Test accuracy of oracle on adversarial examples generated '\n 'using the substitute (excluding originally incorrect labels, NP): ' + str(accuracy))\n accuracies['bbox_on_sub_adv_ex_exc'] = accuracy\n\n return accuracies\n\n\ndef main(argv=None):\n mnist_blackbox(nb_classes=FLAGS.nb_classes, batch_size=FLAGS.batch_size,\n learning_rate=FLAGS.learning_rate,\n nb_epochs=FLAGS.nb_epochs, holdout=FLAGS.holdout,\n data_aug=FLAGS.data_aug, nb_epochs_s=FLAGS.nb_epochs_s,\n lmbda=FLAGS.lmbda, attack=FLAGS.attack, targeted=FLAGS.targeted)\n\n\nif __name__ == '__main__':\n # General flags\n flags.DEFINE_integer('nb_classes', 10, 'Number of classes in problem')\n flags.DEFINE_integer('batch_size', 128, 'Size of training batches')\n flags.DEFINE_integer('n_attack', -1, 'No. of images used for attack')\n flags.DEFINE_float('learning_rate', 0.001, 'Learning rate for training')\n\n # Flags related to oracle\n flags.DEFINE_integer('nb_epochs', 10, 'Number of epochs to train model')\n\n # Flags related to substitute\n flags.DEFINE_integer('holdout', 150, 'Test set holdout for adversary')\n flags.DEFINE_integer('data_aug', 6, 'Nb of substitute data augmentations')\n flags.DEFINE_integer('nb_epochs_s', 30, 'Training epochs for substitute')\n flags.DEFINE_float('lmbda', 0.1, 'Lambda from arxiv.org/abs/1602.02697')\n\n # Flags related to attack\n flags.DEFINE_string('attack', 'cwl2', 'cwl2 = Carlini & Wagner\\'s L2 attack, fgsm = Fast Gradient Sign Method')\n flags.DEFINE_bool('targeted', False, 'use targeted attack')\n\n # Flags related to saving/loading\n flags.DEFINE_bool('load_pretrain', False, 'load pretrained model from sub_saved/mnist-model')\n flags.DEFINE_bool('cached_aug', False, 'use cached augmentation in sub_saved')\n flags.DEFINE_string('train_dir', 'sub_saved', 'model saving path')\n flags.DEFINE_string('filename', 'mnist-model', 'cifar model name')\n\n os.system(\"mkdir -p sub_saved\")\n\n app.run()\n" }, { "path": "retrain.py", "content": "# Copyright 2015 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ==============================================================================\nr\"\"\"Simple transfer learning with Inception v3 or Mobilenet models.\n\nWith support for TensorBoard.\n\nThis example shows how to take a Inception v3 or Mobilenet model trained on\nImageNet images, and train a new top layer that can recognize other classes of\nimages.\n\nThe top layer receives as input a 2048-dimensional vector (1001-dimensional for\nMobilenet) for each image. We train a softmax layer on top of this\nrepresentation. Assuming the softmax layer contains N labels, this corresponds\nto learning N + 2048*N (or 1001*N) model parameters corresponding to the\nlearned biases and weights.\n\nHere's an example, which assumes you have a folder containing class-named\nsubfolders, each full of images for each label. The example folder flower_photos\nshould have a structure like this:\n\n~/flower_photos/daisy/photo1.jpg\n~/flower_photos/daisy/photo2.jpg\n...\n~/flower_photos/rose/anotherphoto77.jpg\n...\n~/flower_photos/sunflower/somepicture.jpg\n\nThe subfolder names are important, since they define what label is applied to\neach image, but the filenames themselves don't matter. Once your images are\nprepared, you can run the training with a command like this:\n\n\n```bash\nbazel build tensorflow/examples/image_retraining:retrain && \\\nbazel-bin/tensorflow/examples/image_retraining/retrain \\\n --image_dir ~/flower_photos\n```\n\nOr, if you have a pip installation of tensorflow, `retrain.py` can be run\nwithout bazel:\n\n```bash\npython tensorflow/examples/image_retraining/retrain.py \\\n --image_dir ~/flower_photos\n```\n\nYou can replace the image_dir argument with any folder containing subfolders of\nimages. The label for each image is taken from the name of the subfolder it's\nin.\n\nThis produces a new model file that can be loaded and run by any TensorFlow\nprogram, for example the label_image sample code.\n\nBy default this script will use the high accuracy, but comparatively large and\nslow Inception v3 model architecture. It's recommended that you start with this\nto validate that you have gathered good training data, but if you want to deploy\non resource-limited platforms, you can try the `--architecture` flag with a\nMobilenet model. For example:\n\n```bash\npython tensorflow/examples/image_retraining/retrain.py \\\n --image_dir ~/flower_photos --architecture mobilenet_1.0_224\n```\n\nThere are 32 different Mobilenet models to choose from, with a variety of file\nsize and latency options. The first number can be '1.0', '0.75', '0.50', or\n'0.25' to control the size, and the second controls the input image size, either\n'224', '192', '160', or '128', with smaller sizes running faster. See\nhttps://research.googleblog.com/2017/06/mobilenets-open-source-models-for.html\nfor more information on Mobilenet.\n\nTo use with TensorBoard:\n\nBy default, this script will log summaries to /tmp/retrain_logs directory\n\nVisualize the summaries with this command:\n\ntensorboard --logdir /tmp/retrain_logs\n\n\"\"\"\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport argparse\nfrom datetime import datetime\nimport hashlib\nimport os.path\nimport random\nimport re\nimport sys\nimport tarfile\n\nimport numpy as np\nfrom six.moves import urllib\nimport tensorflow as tf\n\nfrom tensorflow.python.framework import graph_util\nfrom tensorflow.python.framework import tensor_shape\nfrom tensorflow.python.platform import gfile\nfrom tensorflow.python.util import compat\n\nFLAGS = None\n\n# These are all parameters that are tied to the particular model architecture\n# we're using for Inception v3. These include things like tensor names and their\n# sizes. If you want to adapt this script to work with another model, you will\n# need to update these to reflect the values in the network you're using.\nMAX_NUM_IMAGES_PER_CLASS = 2 ** 27 - 1 # ~134M\n\n\ndef create_image_lists(image_dir, testing_percentage, validation_percentage):\n \"\"\"Builds a list of training images from the file system.\n\n Analyzes the sub folders in the image directory, splits them into stable\n training, testing, and validation sets, and returns a data structure\n describing the lists of images for each label and their paths.\n\n Args:\n image_dir: String path to a folder containing subfolders of images.\n testing_percentage: Integer percentage of the images to reserve for tests.\n validation_percentage: Integer percentage of images reserved for validation.\n\n Returns:\n A dictionary containing an entry for each label subfolder, with images split\n into training, testing, and validation sets within each label.\n \"\"\"\n if not gfile.Exists(image_dir):\n tf.logging.error(\"Image directory '\" + image_dir + \"' not found.\")\n return None\n result = {}\n sub_dirs = [x[0] for x in gfile.Walk(image_dir)]\n # The root directory comes first, so skip it.\n is_root_dir = True\n for sub_dir in sub_dirs:\n if is_root_dir:\n is_root_dir = False\n continue\n extensions = ['jpg', 'jpeg', 'JPG', 'JPEG']\n file_list = []\n dir_name = os.path.basename(sub_dir)\n if dir_name == image_dir:\n continue\n tf.logging.info(\"Looking for images in '\" + dir_name + \"'\")\n for extension in extensions:\n file_glob = os.path.join(image_dir, dir_name, '*.' + extension)\n file_list.extend(gfile.Glob(file_glob))\n if not file_list:\n tf.logging.warning('No files found')\n continue\n if len(file_list) < 20:\n tf.logging.warning(\n 'WARNING: Folder has less than 20 images, which may cause issues.')\n elif len(file_list) > MAX_NUM_IMAGES_PER_CLASS:\n tf.logging.warning(\n 'WARNING: Folder {} has more than {} images. Some images will '\n 'never be selected.'.format(dir_name, MAX_NUM_IMAGES_PER_CLASS))\n label_name = re.sub(r'[^a-z0-9]+', ' ', dir_name.lower())\n training_images = []\n testing_images = []\n validation_images = []\n for file_name in file_list:\n base_name = os.path.basename(file_name)\n # We want to ignore anything after '_nohash_' in the file name when\n # deciding which set to put an image in, the data set creator has a way of\n # grouping photos that are close variations of each other. For example\n # this is used in the plant disease data set to group multiple pictures of\n # the same leaf.\n hash_name = re.sub(r'_nohash_.*$', '', file_name)\n # This looks a bit magical, but we need to decide whether this file should\n # go into the training, testing, or validation sets, and we want to keep\n # existing files in the same set even if more files are subsequently\n # added.\n # To do that, we need a stable way of deciding based on just the file name\n # itself, so we do a hash of that and then use that to generate a\n # probability value that we use to assign it.\n hash_name_hashed = hashlib.sha1(compat.as_bytes(hash_name)).hexdigest()\n percentage_hash = ((int(hash_name_hashed, 16) %\n (MAX_NUM_IMAGES_PER_CLASS + 1)) *\n (100.0 / MAX_NUM_IMAGES_PER_CLASS))\n if percentage_hash < validation_percentage:\n validation_images.append(base_name)\n elif percentage_hash < (testing_percentage + validation_percentage):\n testing_images.append(base_name)\n else:\n training_images.append(base_name)\n result[label_name] = {\n 'dir': dir_name,\n 'training': training_images,\n 'testing': testing_images,\n 'validation': validation_images,\n }\n return result\n\n\ndef get_image_path(image_lists, label_name, index, image_dir, category):\n \"\"\"\"Returns a path to an image for a label at the given index.\n\n Args:\n image_lists: Dictionary of training images for each label.\n label_name: Label string we want to get an image for.\n index: Int offset of the image we want. This will be moduloed by the\n available number of images for the label, so it can be arbitrarily large.\n image_dir: Root folder string of the subfolders containing the training\n images.\n category: Name string of set to pull images from - training, testing, or\n validation.\n\n Returns:\n File system path string to an image that meets the requested parameters.\n\n \"\"\"\n if label_name not in image_lists:\n tf.logging.fatal('Label does not exist %s.', label_name)\n label_lists = image_lists[label_name]\n if category not in label_lists:\n tf.logging.fatal('Category does not exist %s.', category)\n category_list = label_lists[category]\n if not category_list:\n tf.logging.fatal('Label %s has no images in the category %s.',\n label_name, category)\n mod_index = index % len(category_list)\n base_name = category_list[mod_index]\n sub_dir = label_lists['dir']\n full_path = os.path.join(image_dir, sub_dir, base_name)\n return full_path\n\n\ndef get_bottleneck_path(image_lists, label_name, index, bottleneck_dir,\n category, architecture):\n \"\"\"\"Returns a path to a bottleneck file for a label at the given index.\n\n Args:\n image_lists: Dictionary of training images for each label.\n label_name: Label string we want to get an image for.\n index: Integer offset of the image we want. This will be moduloed by the\n available number of images for the label, so it can be arbitrarily large.\n bottleneck_dir: Folder string holding cached files of bottleneck values.\n category: Name string of set to pull images from - training, testing, or\n validation.\n architecture: The name of the model architecture.\n\n Returns:\n File system path string to an image that meets the requested parameters.\n \"\"\"\n return get_image_path(image_lists, label_name, index, bottleneck_dir,\n category) + '_' + architecture + '.txt'\n\n\ndef create_model_graph(model_info):\n \"\"\"\"Creates a graph from saved GraphDef file and returns a Graph object.\n\n Args:\n model_info: Dictionary containing information about the model architecture.\n\n Returns:\n Graph holding the trained Inception network, and various tensors we'll be\n manipulating.\n \"\"\"\n with tf.Graph().as_default() as graph:\n model_path = os.path.join(FLAGS.model_dir, model_info['model_file_name'])\n with gfile.FastGFile(model_path, 'rb') as f:\n graph_def = tf.GraphDef()\n graph_def.ParseFromString(f.read())\n bottleneck_tensor, resized_input_tensor = (tf.import_graph_def(\n graph_def,\n name='',\n return_elements=[\n model_info['bottleneck_tensor_name'],\n model_info['resized_input_tensor_name'],\n ]))\n return graph, bottleneck_tensor, resized_input_tensor\n\n\ndef run_bottleneck_on_image(sess, image_data, image_data_tensor,\n decoded_image_tensor, resized_input_tensor,\n bottleneck_tensor):\n \"\"\"Runs inference on an image to extract the 'bottleneck' summary layer.\n\n Args:\n sess: Current active TensorFlow Session.\n image_data: String of raw JPEG data.\n image_data_tensor: Input data layer in the graph.\n decoded_image_tensor: Output of initial image resizing and preprocessing.\n resized_input_tensor: The input node of the recognition graph.\n bottleneck_tensor: Layer before the final softmax.\n\n Returns:\n Numpy array of bottleneck values.\n \"\"\"\n # First decode the JPEG image, resize it, and rescale the pixel values.\n resized_input_values = sess.run(decoded_image_tensor,\n {image_data_tensor: image_data})\n # Then run it through the recognition network.\n bottleneck_values = sess.run(bottleneck_tensor,\n {resized_input_tensor: resized_input_values})\n bottleneck_values = np.squeeze(bottleneck_values)\n return bottleneck_values\n\n\ndef maybe_download_and_extract(data_url):\n \"\"\"Download and extract model tar file.\n\n If the pretrained model we're using doesn't already exist, this function\n downloads it from the TensorFlow.org website and unpacks it into a directory.\n\n Args:\n data_url: Web location of the tar file containing the pretrained model.\n \"\"\"\n dest_directory = FLAGS.model_dir\n if not os.path.exists(dest_directory):\n os.makedirs(dest_directory)\n filename = data_url.split('/')[-1]\n filepath = os.path.join(dest_directory, filename)\n if not os.path.exists(filepath):\n\n def _progress(count, block_size, total_size):\n sys.stdout.write('\\r>> Downloading %s %.1f%%' %\n (filename,\n float(count * block_size) / float(total_size) * 100.0))\n sys.stdout.flush()\n\n filepath, _ = urllib.request.urlretrieve(data_url, filepath, _progress)\n print()\n statinfo = os.stat(filepath)\n tf.logging.info('Successfully downloaded', filename, statinfo.st_size,\n 'bytes.')\n tarfile.open(filepath, 'r:gz').extractall(dest_directory)\n\n\ndef ensure_dir_exists(dir_name):\n \"\"\"Makes sure the folder exists on disk.\n\n Args:\n dir_name: Path string to the folder we want to create.\n \"\"\"\n if not os.path.exists(dir_name):\n os.makedirs(dir_name)\n\n\nbottleneck_path_2_bottleneck_values = {}\n\n\ndef create_bottleneck_file(bottleneck_path, image_lists, label_name, index,\n image_dir, category, sess, jpeg_data_tensor,\n decoded_image_tensor, resized_input_tensor,\n bottleneck_tensor):\n \"\"\"Create a single bottleneck file.\"\"\"\n tf.logging.info('Creating bottleneck at ' + bottleneck_path)\n image_path = get_image_path(image_lists, label_name, index,\n image_dir, category)\n if not gfile.Exists(image_path):\n tf.logging.fatal('File does not exist %s', image_path)\n image_data = gfile.FastGFile(image_path, 'rb').read()\n try:\n bottleneck_values = run_bottleneck_on_image(\n sess, image_data, jpeg_data_tensor, decoded_image_tensor,\n resized_input_tensor, bottleneck_tensor)\n except Exception as e:\n raise RuntimeError('Error during processing file %s (%s)' % (image_path,\n str(e)))\n bottleneck_string = ','.join(str(x) for x in bottleneck_values)\n with open(bottleneck_path, 'w') as bottleneck_file:\n bottleneck_file.write(bottleneck_string)\n\n\ndef get_or_create_bottleneck(sess, image_lists, label_name, index, image_dir,\n category, bottleneck_dir, jpeg_data_tensor,\n decoded_image_tensor, resized_input_tensor,\n bottleneck_tensor, architecture):\n \"\"\"Retrieves or calculates bottleneck values for an image.\n\n If a cached version of the bottleneck data exists on-disk, return that,\n otherwise calculate the data and save it to disk for future use.\n\n Args:\n sess: The current active TensorFlow Session.\n image_lists: Dictionary of training images for each label.\n label_name: Label string we want to get an image for.\n index: Integer offset of the image we want. This will be modulo-ed by the\n available number of images for the label, so it can be arbitrarily large.\n image_dir: Root folder string of the subfolders containing the training\n images.\n category: Name string of which set to pull images from - training, testing,\n or validation.\n bottleneck_dir: Folder string holding cached files of bottleneck values.\n jpeg_data_tensor: The tensor to feed loaded jpeg data into.\n decoded_image_tensor: The output of decoding and resizing the image.\n resized_input_tensor: The input node of the recognition graph.\n bottleneck_tensor: The output tensor for the bottleneck values.\n architecture: The name of the model architecture.\n\n Returns:\n Numpy array of values produced by the bottleneck layer for the image.\n \"\"\"\n label_lists = image_lists[label_name]\n sub_dir = label_lists['dir']\n sub_dir_path = os.path.join(bottleneck_dir, sub_dir)\n ensure_dir_exists(sub_dir_path)\n bottleneck_path = get_bottleneck_path(image_lists, label_name, index,\n bottleneck_dir, category, architecture)\n if not os.path.exists(bottleneck_path):\n create_bottleneck_file(bottleneck_path, image_lists, label_name, index,\n image_dir, category, sess, jpeg_data_tensor,\n decoded_image_tensor, resized_input_tensor,\n bottleneck_tensor)\n with open(bottleneck_path, 'r') as bottleneck_file:\n bottleneck_string = bottleneck_file.read()\n did_hit_error = False\n try:\n bottleneck_values = [float(x) for x in bottleneck_string.split(',')]\n except ValueError:\n tf.logging.warning('Invalid float found, recreating bottleneck')\n did_hit_error = True\n if did_hit_error:\n create_bottleneck_file(bottleneck_path, image_lists, label_name, index,\n image_dir, category, sess, jpeg_data_tensor,\n decoded_image_tensor, resized_input_tensor,\n bottleneck_tensor)\n with open(bottleneck_path, 'r') as bottleneck_file:\n bottleneck_string = bottleneck_file.read()\n # Allow exceptions to propagate here, since they shouldn't happen after a\n # fresh creation\n bottleneck_values = [float(x) for x in bottleneck_string.split(',')]\n return bottleneck_values\n\n\ndef cache_bottlenecks(sess, image_lists, image_dir, bottleneck_dir,\n jpeg_data_tensor, decoded_image_tensor,\n resized_input_tensor, bottleneck_tensor, architecture):\n \"\"\"Ensures all the training, testing, and validation bottlenecks are cached.\n\n Because we're likely to read the same image multiple times (if there are no\n distortions applied during training) it can speed things up a lot if we\n calculate the bottleneck layer values once for each image during\n preprocessing, and then just read those cached values repeatedly during\n training. Here we go through all the images we've found, calculate those\n values, and save them off.\n\n Args:\n sess: The current active TensorFlow Session.\n image_lists: Dictionary of training images for each label.\n image_dir: Root folder string of the subfolders containing the training\n images.\n bottleneck_dir: Folder string holding cached files of bottleneck values.\n jpeg_data_tensor: Input tensor for jpeg data from file.\n decoded_image_tensor: The output of decoding and resizing the image.\n resized_input_tensor: The input node of the recognition graph.\n bottleneck_tensor: The penultimate output layer of the graph.\n architecture: The name of the model architecture.\n\n Returns:\n Nothing.\n \"\"\"\n how_many_bottlenecks = 0\n ensure_dir_exists(bottleneck_dir)\n for label_name, label_lists in image_lists.items():\n for category in ['training', 'testing', 'validation']:\n category_list = label_lists[category]\n for index, unused_base_name in enumerate(category_list):\n get_or_create_bottleneck(\n sess, image_lists, label_name, index, image_dir, category,\n bottleneck_dir, jpeg_data_tensor, decoded_image_tensor,\n resized_input_tensor, bottleneck_tensor, architecture)\n\n how_many_bottlenecks += 1\n if how_many_bottlenecks % 100 == 0:\n tf.logging.info(\n str(how_many_bottlenecks) + ' bottleneck files created.')\n\n\ndef get_random_cached_bottlenecks(sess, image_lists, how_many, category,\n bottleneck_dir, image_dir, jpeg_data_tensor,\n decoded_image_tensor, resized_input_tensor,\n bottleneck_tensor, architecture):\n \"\"\"Retrieves bottleneck values for cached images.\n\n If no distortions are being applied, this function can retrieve the cached\n bottleneck values directly from disk for images. It picks a random set of\n images from the specified category.\n\n Args:\n sess: Current TensorFlow Session.\n image_lists: Dictionary of training images for each label.\n how_many: If positive, a random sample of this size will be chosen.\n If negative, all bottlenecks will be retrieved.\n category: Name string of which set to pull from - training, testing, or\n validation.\n bottleneck_dir: Folder string holding cached files of bottleneck values.\n image_dir: Root folder string of the subfolders containing the training\n images.\n jpeg_data_tensor: The layer to feed jpeg image data into.\n decoded_image_tensor: The output of decoding and resizing the image.\n resized_input_tensor: The input node of the recognition graph.\n bottleneck_tensor: The bottleneck output layer of the CNN graph.\n architecture: The name of the model architecture.\n\n Returns:\n List of bottleneck arrays, their corresponding ground truths, and the\n relevant filenames.\n \"\"\"\n class_count = len(image_lists.keys())\n bottlenecks = []\n ground_truths = []\n filenames = []\n if how_many >= 0:\n # Retrieve a random sample of bottlenecks.\n for unused_i in range(how_many):\n label_index = random.randrange(class_count)\n label_name = list(image_lists.keys())[label_index]\n image_index = random.randrange(MAX_NUM_IMAGES_PER_CLASS + 1)\n image_name = get_image_path(image_lists, label_name, image_index,\n image_dir, category)\n bottleneck = get_or_create_bottleneck(\n sess, image_lists, label_name, image_index, image_dir, category,\n bottleneck_dir, jpeg_data_tensor, decoded_image_tensor,\n resized_input_tensor, bottleneck_tensor, architecture)\n ground_truth = np.zeros(class_count, dtype=np.float32)\n ground_truth[label_index] = 1.0\n bottlenecks.append(bottleneck)\n ground_truths.append(ground_truth)\n filenames.append(image_name)\n else:\n # Retrieve all bottlenecks.\n for label_index, label_name in enumerate(image_lists.keys()):\n for image_index, image_name in enumerate(\n image_lists[label_name][category]):\n image_name = get_image_path(image_lists, label_name, image_index,\n image_dir, category)\n bottleneck = get_or_create_bottleneck(\n sess, image_lists, label_name, image_index, image_dir, category,\n bottleneck_dir, jpeg_data_tensor, decoded_image_tensor,\n resized_input_tensor, bottleneck_tensor, architecture)\n ground_truth = np.zeros(class_count, dtype=np.float32)\n ground_truth[label_index] = 1.0\n bottlenecks.append(bottleneck)\n ground_truths.append(ground_truth)\n filenames.append(image_name)\n return bottlenecks, ground_truths, filenames\n\n\ndef get_random_distorted_bottlenecks(\n sess, image_lists, how_many, category, image_dir, input_jpeg_tensor,\n distorted_image, resized_input_tensor, bottleneck_tensor):\n \"\"\"Retrieves bottleneck values for training images, after distortions.\n\n If we're training with distortions like crops, scales, or flips, we have to\n recalculate the full model for every image, and so we can't use cached\n bottleneck values. Instead we find random images for the requested category,\n run them through the distortion graph, and then the full graph to get the\n bottleneck results for each.\n\n Args:\n sess: Current TensorFlow Session.\n image_lists: Dictionary of training images for each label.\n how_many: The integer number of bottleneck values to return.\n category: Name string of which set of images to fetch - training, testing,\n or validation.\n image_dir: Root folder string of the subfolders containing the training\n images.\n input_jpeg_tensor: The input layer we feed the image data to.\n distorted_image: The output node of the distortion graph.\n resized_input_tensor: The input node of the recognition graph.\n bottleneck_tensor: The bottleneck output layer of the CNN graph.\n\n Returns:\n List of bottleneck arrays and their corresponding ground truths.\n \"\"\"\n class_count = len(image_lists.keys())\n bottlenecks = []\n ground_truths = []\n for unused_i in range(how_many):\n label_index = random.randrange(class_count)\n label_name = list(image_lists.keys())[label_index]\n image_index = random.randrange(MAX_NUM_IMAGES_PER_CLASS + 1)\n image_path = get_image_path(image_lists, label_name, image_index, image_dir,\n category)\n if not gfile.Exists(image_path):\n tf.logging.fatal('File does not exist %s', image_path)\n jpeg_data = gfile.FastGFile(image_path, 'rb').read()\n # Note that we materialize the distorted_image_data as a numpy array before\n # sending running inference on the image. This involves 2 memory copies and\n # might be optimized in other implementations.\n distorted_image_data = sess.run(distorted_image,\n {input_jpeg_tensor: jpeg_data})\n bottleneck_values = sess.run(bottleneck_tensor,\n {resized_input_tensor: distorted_image_data})\n bottleneck_values = np.squeeze(bottleneck_values)\n ground_truth = np.zeros(class_count, dtype=np.float32)\n ground_truth[label_index] = 1.0\n bottlenecks.append(bottleneck_values)\n ground_truths.append(ground_truth)\n return bottlenecks, ground_truths\n\n\ndef should_distort_images(flip_left_right, random_crop, random_scale,\n random_brightness):\n \"\"\"Whether any distortions are enabled, from the input flags.\n\n Args:\n flip_left_right: Boolean whether to randomly mirror images horizontally.\n random_crop: Integer percentage setting the total margin used around the\n crop box.\n random_scale: Integer percentage of how much to vary the scale by.\n random_brightness: Integer range to randomly multiply the pixel values by.\n\n Returns:\n Boolean value indicating whether any distortions should be applied.\n \"\"\"\n return (flip_left_right or (random_crop != 0) or (random_scale != 0) or\n (random_brightness != 0))\n\n\ndef add_input_distortions(flip_left_right, random_crop, random_scale,\n random_brightness, input_width, input_height,\n input_depth, input_mean, input_std):\n \"\"\"Creates the operations to apply the specified distortions.\n\n During training it can help to improve the results if we run the images\n through simple distortions like crops, scales, and flips. These reflect the\n kind of variations we expect in the real world, and so can help train the\n model to cope with natural data more effectively. Here we take the supplied\n parameters and construct a network of operations to apply them to an image.\n\n Cropping\n ~~~~~~~~\n\n Cropping is done by placing a bounding box at a random position in the full\n image. The cropping parameter controls the size of that box relative to the\n input image. If it's zero, then the box is the same size as the input and no\n cropping is performed. If the value is 50%, then the crop box will be half the\n width and height of the input. In a diagram it looks like this:\n\n < width >\n +---------------------+\n | |\n | width - crop% |\n | < > |\n | +------+ |\n | | | |\n | | | |\n | | | |\n | +------+ |\n | |\n | |\n +---------------------+\n\n Scaling\n ~~~~~~~\n\n Scaling is a lot like cropping, except that the bounding box is always\n centered and its size varies randomly within the given range. For example if\n the scale percentage is zero, then the bounding box is the same size as the\n input and no scaling is applied. If it's 50%, then the bounding box will be in\n a random range between half the width and height and full size.\n\n Args:\n flip_left_right: Boolean whether to randomly mirror images horizontally.\n random_crop: Integer percentage setting the total margin used around the\n crop box.\n random_scale: Integer percentage of how much to vary the scale by.\n random_brightness: Integer range to randomly multiply the pixel values by.\n graph.\n input_width: Horizontal size of expected input image to model.\n input_height: Vertical size of expected input image to model.\n input_depth: How many channels the expected input image should have.\n input_mean: Pixel value that should be zero in the image for the graph.\n input_std: How much to divide the pixel values by before recognition.\n\n Returns:\n The jpeg input layer and the distorted result tensor.\n \"\"\"\n\n jpeg_data = tf.placeholder(tf.string, name='DistortJPGInput')\n decoded_image = tf.image.decode_jpeg(jpeg_data, channels=input_depth)\n decoded_image_as_float = tf.cast(decoded_image, dtype=tf.float32)\n decoded_image_4d = tf.expand_dims(decoded_image_as_float, 0)\n margin_scale = 1.0 + (random_crop / 100.0)\n resize_scale = 1.0 + (random_scale / 100.0)\n margin_scale_value = tf.constant(margin_scale)\n resize_scale_value = tf.random_uniform(tensor_shape.scalar(),\n minval=1.0,\n maxval=resize_scale)\n scale_value = tf.multiply(margin_scale_value, resize_scale_value)\n precrop_width = tf.multiply(scale_value, input_width)\n precrop_height = tf.multiply(scale_value, input_height)\n precrop_shape = tf.stack([precrop_height, precrop_width])\n precrop_shape_as_int = tf.cast(precrop_shape, dtype=tf.int32)\n precropped_image = tf.image.resize_bilinear(decoded_image_4d,\n precrop_shape_as_int)\n precropped_image_3d = tf.squeeze(precropped_image, squeeze_dims=[0])\n cropped_image = tf.random_crop(precropped_image_3d,\n [input_height, input_width, input_depth])\n if flip_left_right:\n flipped_image = tf.image.random_flip_left_right(cropped_image)\n else:\n flipped_image = cropped_image\n brightness_min = 1.0 - (random_brightness / 100.0)\n brightness_max = 1.0 + (random_brightness / 100.0)\n brightness_value = tf.random_uniform(tensor_shape.scalar(),\n minval=brightness_min,\n maxval=brightness_max)\n brightened_image = tf.multiply(flipped_image, brightness_value)\n offset_image = tf.subtract(brightened_image, input_mean)\n mul_image = tf.multiply(offset_image, 1.0 / input_std)\n distort_result = tf.expand_dims(mul_image, 0, name='DistortResult')\n return jpeg_data, distort_result\n\n\ndef variable_summaries(var):\n \"\"\"Attach a lot of summaries to a Tensor (for TensorBoard visualization).\"\"\"\n with tf.name_scope('summaries'):\n mean = tf.reduce_mean(var)\n tf.summary.scalar('mean', mean)\n with tf.name_scope('stddev'):\n stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))\n tf.summary.scalar('stddev', stddev)\n tf.summary.scalar('max', tf.reduce_max(var))\n tf.summary.scalar('min', tf.reduce_min(var))\n tf.summary.histogram('histogram', var)\n\n\ndef add_final_training_ops(class_count, final_tensor_name, bottleneck_tensor,\n bottleneck_tensor_size):\n \"\"\"Adds a new softmax and fully-connected layer for training.\n\n We need to retrain the top layer to identify our new classes, so this function\n adds the right operations to the graph, along with some variables to hold the\n weights, and then sets up all the gradients for the backward pass.\n\n The set up for the softmax and fully-connected layers is based on:\n https://www.tensorflow.org/versions/master/tutorials/mnist/beginners/index.html\n\n Args:\n class_count: Integer of how many categories of things we're trying to\n recognize.\n final_tensor_name: Name string for the new final node that produces results.\n bottleneck_tensor: The output of the main CNN graph.\n bottleneck_tensor_size: How many entries in the bottleneck vector.\n\n Returns:\n The tensors for the training and cross entropy results, and tensors for the\n bottleneck input and ground truth input.\n \"\"\"\n with tf.name_scope('input'):\n bottleneck_input = tf.placeholder_with_default(\n bottleneck_tensor,\n shape=[None, bottleneck_tensor_size],\n name='BottleneckInputPlaceholder')\n\n ground_truth_input = tf.placeholder(tf.float32,\n [None, class_count],\n name='GroundTruthInput')\n\n # Organizing the following ops as `final_training_ops` so they're easier\n # to see in TensorBoard\n layer_name = 'final_training_ops'\n with tf.name_scope(layer_name):\n with tf.name_scope('weights'):\n initial_value = tf.truncated_normal(\n [bottleneck_tensor_size, class_count], stddev=0.001)\n\n layer_weights = tf.Variable(initial_value, name='final_weights')\n\n variable_summaries(layer_weights)\n with tf.name_scope('biases'):\n layer_biases = tf.Variable(tf.zeros([class_count]), name='final_biases')\n variable_summaries(layer_biases)\n with tf.name_scope('Wx_plus_b'):\n logits = tf.matmul(bottleneck_input, layer_weights) + layer_biases\n tf.summary.histogram('pre_activations', logits)\n\n final_tensor = tf.nn.softmax(logits, name=final_tensor_name)\n tf.summary.histogram('activations', final_tensor)\n\n with tf.name_scope('cross_entropy'):\n cross_entropy = tf.nn.softmax_cross_entropy_with_logits(\n labels=ground_truth_input, logits=logits)\n with tf.name_scope('total'):\n cross_entropy_mean = tf.reduce_mean(cross_entropy)\n tf.summary.scalar('cross_entropy', cross_entropy_mean)\n\n with tf.name_scope('train'):\n optimizer = tf.train.GradientDescentOptimizer(FLAGS.learning_rate)\n train_step = optimizer.minimize(cross_entropy_mean)\n\n return (train_step, cross_entropy_mean, bottleneck_input, ground_truth_input,\n final_tensor)\n\n\ndef add_evaluation_step(result_tensor, ground_truth_tensor):\n \"\"\"Inserts the operations we need to evaluate the accuracy of our results.\n\n Args:\n result_tensor: The new final node that produces results.\n ground_truth_tensor: The node we feed ground truth data\n into.\n\n Returns:\n Tuple of (evaluation step, prediction).\n \"\"\"\n with tf.name_scope('accuracy'):\n with tf.name_scope('correct_prediction'):\n prediction = tf.argmax(result_tensor, 1)\n correct_prediction = tf.equal(\n prediction, tf.argmax(ground_truth_tensor, 1))\n with tf.name_scope('accuracy'):\n evaluation_step = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n tf.summary.scalar('accuracy', evaluation_step)\n return evaluation_step, prediction\n\n\ndef save_graph_to_file(sess, graph, graph_file_name):\n output_graph_def = graph_util.convert_variables_to_constants(\n sess, graph.as_graph_def(), [FLAGS.final_tensor_name])\n with gfile.FastGFile(graph_file_name, 'wb') as f:\n f.write(output_graph_def.SerializeToString())\n return\n\n\ndef prepare_file_system():\n # Setup the directory we'll write summaries to for TensorBoard\n if tf.gfile.Exists(FLAGS.summaries_dir):\n tf.gfile.DeleteRecursively(FLAGS.summaries_dir)\n tf.gfile.MakeDirs(FLAGS.summaries_dir)\n if FLAGS.intermediate_store_frequency > 0:\n ensure_dir_exists(FLAGS.intermediate_output_graphs_dir)\n return\n\n\ndef create_model_info(architecture):\n \"\"\"Given the name of a model architecture, returns information about it.\n\n There are different base image recognition pretrained models that can be\n retrained using transfer learning, and this function translates from the name\n of a model to the attributes that are needed to download and train with it.\n\n Args:\n architecture: Name of a model architecture.\n\n Returns:\n Dictionary of information about the model, or None if the name isn't\n recognized\n\n Raises:\n ValueError: If architecture name is unknown.\n \"\"\"\n architecture = architecture.lower()\n if architecture == 'inception_v3':\n # pylint: disable=line-too-long\n data_url = 'http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz'\n # pylint: enable=line-too-long\n bottleneck_tensor_name = 'pool_3/_reshape:0'\n bottleneck_tensor_size = 2048\n input_width = 299\n input_height = 299\n input_depth = 3\n resized_input_tensor_name = 'Mul:0'\n model_file_name = 'classify_image_graph_def.pb'\n input_mean = 128\n input_std = 128\n elif architecture.startswith('mobilenet_'):\n parts = architecture.split('_')\n if len(parts) != 3 and len(parts) != 4:\n tf.logging.error(\"Couldn't understand architecture name '%s'\",\n architecture)\n return None\n version_string = parts[1]\n if (version_string != '1.0' and version_string != '0.75' and\n version_string != '0.50' and version_string != '0.25'):\n tf.logging.error(\n \"\"\"\"The Mobilenet version should be '1.0', '0.75', '0.50', or '0.25',\n but found '%s' for architecture '%s'\"\"\",\n version_string, architecture)\n return None\n size_string = parts[2]\n if (size_string != '224' and size_string != '192' and\n size_string != '160' and size_string != '128'):\n tf.logging.error(\n \"\"\"The Mobilenet input size should be '224', '192', '160', or '128',\n but found '%s' for architecture '%s'\"\"\",\n size_string, architecture)\n return None\n if len(parts) == 3:\n is_quantized = False\n else:\n if parts[3] != 'quantized':\n tf.logging.error(\n \"Couldn't understand architecture suffix '%s' for '%s'\", parts[3],\n architecture)\n return None\n is_quantized = True\n data_url = 'http://download.tensorflow.org/models/mobilenet_v1_'\n data_url += version_string + '_' + size_string + '_frozen.tgz'\n bottleneck_tensor_name = 'MobilenetV1/Predictions/Reshape:0'\n bottleneck_tensor_size = 1001\n input_width = int(size_string)\n input_height = int(size_string)\n input_depth = 3\n resized_input_tensor_name = 'input:0'\n if is_quantized:\n model_base_name = 'quantized_graph.pb'\n else:\n model_base_name = 'frozen_graph.pb'\n model_dir_name = 'mobilenet_v1_' + version_string + '_' + size_string\n model_file_name = os.path.join(model_dir_name, model_base_name)\n input_mean = 127.5\n input_std = 127.5\n else:\n tf.logging.error(\"Couldn't understand architecture name '%s'\", architecture)\n raise ValueError('Unknown architecture', architecture)\n\n return {\n 'data_url': data_url,\n 'bottleneck_tensor_name': bottleneck_tensor_name,\n 'bottleneck_tensor_size': bottleneck_tensor_size,\n 'input_width': input_width,\n 'input_height': input_height,\n 'input_depth': input_depth,\n 'resized_input_tensor_name': resized_input_tensor_name,\n 'model_file_name': model_file_name,\n 'input_mean': input_mean,\n 'input_std': input_std,\n }\n\n\ndef add_jpeg_decoding(input_width, input_height, input_depth, input_mean,\n input_std):\n \"\"\"Adds operations that perform JPEG decoding and resizing to the graph..\n\n Args:\n input_width: Desired width of the image fed into the recognizer graph.\n input_height: Desired width of the image fed into the recognizer graph.\n input_depth: Desired channels of the image fed into the recognizer graph.\n input_mean: Pixel value that should be zero in the image for the graph.\n input_std: How much to divide the pixel values by before recognition.\n\n Returns:\n Tensors for the node to feed JPEG data into, and the output of the\n preprocessing steps.\n \"\"\"\n jpeg_data = tf.placeholder(tf.string, name='DecodeJPGInput')\n decoded_image = tf.image.decode_jpeg(jpeg_data, channels=input_depth)\n decoded_image_as_float = tf.cast(decoded_image, dtype=tf.float32)\n decoded_image_4d = tf.expand_dims(decoded_image_as_float, 0)\n resize_shape = tf.stack([input_height, input_width])\n resize_shape_as_int = tf.cast(resize_shape, dtype=tf.int32)\n resized_image = tf.image.resize_bilinear(decoded_image_4d,\n resize_shape_as_int)\n offset_image = tf.subtract(resized_image, input_mean)\n mul_image = tf.multiply(offset_image, 1.0 / input_std)\n return jpeg_data, mul_image\n\n\ndef main(_):\n # Needed to make sure the logging output is visible.\n # See https://github.com/tensorflow/tensorflow/issues/3047\n tf.logging.set_verbosity(tf.logging.INFO)\n\n # Prepare necessary directories that can be used during training\n prepare_file_system()\n\n # Gather information about the model architecture we'll be using.\n model_info = create_model_info(FLAGS.architecture)\n if not model_info:\n tf.logging.error('Did not recognize architecture flag')\n return -1\n\n # Set up the pre-trained graph.\n maybe_download_and_extract(model_info['data_url'])\n graph, bottleneck_tensor, resized_image_tensor = (\n create_model_graph(model_info))\n\n # Look at the folder structure, and create lists of all the images.\n image_lists = create_image_lists(FLAGS.image_dir, FLAGS.testing_percentage,\n FLAGS.validation_percentage)\n class_count = len(image_lists.keys())\n if class_count == 0:\n tf.logging.error('No valid folders of images found at ' + FLAGS.image_dir)\n return -1\n if class_count == 1:\n tf.logging.error('Only one valid folder of images found at ' +\n FLAGS.image_dir +\n ' - multiple classes are needed for classification.')\n return -1\n\n # See if the command-line flags mean we're applying any distortions.\n do_distort_images = should_distort_images(\n FLAGS.flip_left_right, FLAGS.random_crop, FLAGS.random_scale,\n FLAGS.random_brightness)\n\n with tf.Session(graph=graph) as sess:\n # Set up the image decoding sub-graph.\n jpeg_data_tensor, decoded_image_tensor = add_jpeg_decoding(\n model_info['input_width'], model_info['input_height'],\n model_info['input_depth'], model_info['input_mean'],\n model_info['input_std'])\n\n if do_distort_images:\n # We will be applying distortions, so setup the operations we'll need.\n (distorted_jpeg_data_tensor,\n distorted_image_tensor) = add_input_distortions(\n FLAGS.flip_left_right, FLAGS.random_crop, FLAGS.random_scale,\n FLAGS.random_brightness, model_info['input_width'],\n model_info['input_height'], model_info['input_depth'],\n model_info['input_mean'], model_info['input_std'])\n else:\n # We'll make sure we've calculated the 'bottleneck' image summaries and\n # cached them on disk.\n cache_bottlenecks(sess, image_lists, FLAGS.image_dir,\n FLAGS.bottleneck_dir, jpeg_data_tensor,\n decoded_image_tensor, resized_image_tensor,\n bottleneck_tensor, FLAGS.architecture)\n\n # Add the new layer that we'll be training.\n (train_step, cross_entropy, bottleneck_input, ground_truth_input,\n final_tensor) = add_final_training_ops(\n len(image_lists.keys()), FLAGS.final_tensor_name, bottleneck_tensor,\n model_info['bottleneck_tensor_size'])\n\n # Create the operations we need to evaluate the accuracy of our new layer.\n evaluation_step, prediction = add_evaluation_step(\n final_tensor, ground_truth_input)\n\n # Merge all the summaries and write them out to the summaries_dir\n merged = tf.summary.merge_all()\n train_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/train',\n sess.graph)\n\n validation_writer = tf.summary.FileWriter(\n FLAGS.summaries_dir + '/validation')\n\n # Set up all our weights to their initial default values.\n init = tf.global_variables_initializer()\n sess.run(init)\n\n # Run the training for as many cycles as requested on the command line.\n for i in range(FLAGS.how_many_training_steps):\n # Get a batch of input bottleneck values, either calculated fresh every\n # time with distortions applied, or from the cache stored on disk.\n if do_distort_images:\n (train_bottlenecks,\n train_ground_truth) = get_random_distorted_bottlenecks(\n sess, image_lists, FLAGS.train_batch_size, 'training',\n FLAGS.image_dir, distorted_jpeg_data_tensor,\n distorted_image_tensor, resized_image_tensor, bottleneck_tensor)\n else:\n (train_bottlenecks,\n train_ground_truth, _) = get_random_cached_bottlenecks(\n sess, image_lists, FLAGS.train_batch_size, 'training',\n FLAGS.bottleneck_dir, FLAGS.image_dir, jpeg_data_tensor,\n decoded_image_tensor, resized_image_tensor, bottleneck_tensor,\n FLAGS.architecture)\n # Feed the bottlenecks and ground truth into the graph, and run a training\n # step. Capture training summaries for TensorBoard with the `merged` op.\n train_summary, _ = sess.run(\n [merged, train_step],\n feed_dict={bottleneck_input: train_bottlenecks,\n ground_truth_input: train_ground_truth})\n train_writer.add_summary(train_summary, i)\n\n # Every so often, print out how well the graph is training.\n is_last_step = (i + 1 == FLAGS.how_many_training_steps)\n if (i % FLAGS.eval_step_interval) == 0 or is_last_step:\n train_accuracy, cross_entropy_value = sess.run(\n [evaluation_step, cross_entropy],\n feed_dict={bottleneck_input: train_bottlenecks,\n ground_truth_input: train_ground_truth})\n tf.logging.info('%s: Step %d: Train accuracy = %.1f%%' %\n (datetime.now(), i, train_accuracy * 100))\n tf.logging.info('%s: Step %d: Cross entropy = %f' %\n (datetime.now(), i, cross_entropy_value))\n validation_bottlenecks, validation_ground_truth, _ = (\n get_random_cached_bottlenecks(\n sess, image_lists, FLAGS.validation_batch_size, 'validation',\n FLAGS.bottleneck_dir, FLAGS.image_dir, jpeg_data_tensor,\n decoded_image_tensor, resized_image_tensor, bottleneck_tensor,\n FLAGS.architecture))\n # Run a validation step and capture training summaries for TensorBoard\n # with the `merged` op.\n validation_summary, validation_accuracy = sess.run(\n [merged, evaluation_step],\n feed_dict={bottleneck_input: validation_bottlenecks,\n ground_truth_input: validation_ground_truth})\n validation_writer.add_summary(validation_summary, i)\n tf.logging.info('%s: Step %d: Validation accuracy = %.1f%% (N=%d)' %\n (datetime.now(), i, validation_accuracy * 100,\n len(validation_bottlenecks)))\n\n # Store intermediate results\n intermediate_frequency = FLAGS.intermediate_store_frequency\n\n if (intermediate_frequency > 0 and (i % intermediate_frequency == 0)\n and i > 0):\n intermediate_file_name = (FLAGS.intermediate_output_graphs_dir +\n 'intermediate_' + str(i) + '.pb')\n tf.logging.info('Save intermediate result to : ' +\n intermediate_file_name)\n save_graph_to_file(sess, graph, intermediate_file_name)\n\n # We've completed all our training, so run a final test evaluation on\n # some new images we haven't used before.\n test_bottlenecks, test_ground_truth, test_filenames = (\n get_random_cached_bottlenecks(\n sess, image_lists, FLAGS.test_batch_size, 'testing',\n FLAGS.bottleneck_dir, FLAGS.image_dir, jpeg_data_tensor,\n decoded_image_tensor, resized_image_tensor, bottleneck_tensor,\n FLAGS.architecture))\n test_accuracy, predictions = sess.run(\n [evaluation_step, prediction],\n feed_dict={bottleneck_input: test_bottlenecks,\n ground_truth_input: test_ground_truth})\n tf.logging.info('Final test accuracy = %.1f%% (N=%d)' %\n (test_accuracy * 100, len(test_bottlenecks)))\n\n if FLAGS.print_misclassified_test_images:\n tf.logging.info('=== MISCLASSIFIED TEST IMAGES ===')\n for i, test_filename in enumerate(test_filenames):\n if predictions[i] != test_ground_truth[i].argmax():\n tf.logging.info('%70s %s' %\n (test_filename,\n list(image_lists.keys())[predictions[i]]))\n\n # Write out the trained graph and labels with the weights stored as\n # constants.\n save_graph_to_file(sess, graph, FLAGS.output_graph)\n with gfile.FastGFile(FLAGS.output_labels, 'w') as f:\n f.write('\\n'.join(image_lists.keys()) + '\\n')\n\n\nif __name__ == '__main__':\n parser = argparse.ArgumentParser()\n parser.add_argument(\n '--image_dir',\n type=str,\n default='',\n help='Path to folders of labeled images.'\n )\n parser.add_argument(\n '--output_graph',\n type=str,\n default='/tmp/output_graph.pb',\n help='Where to save the trained graph.'\n )\n parser.add_argument(\n '--intermediate_output_graphs_dir',\n type=str,\n default='/tmp/intermediate_graph/',\n help='Where to save the intermediate graphs.'\n )\n parser.add_argument(\n '--intermediate_store_frequency',\n type=int,\n default=0,\n help=\"\"\"\\\n How many steps to store intermediate graph. If \"0\" then will not\n store.\\\n \"\"\"\n )\n parser.add_argument(\n '--output_labels',\n type=str,\n default='/tmp/output_labels.txt',\n help='Where to save the trained graph\\'s labels.'\n )\n parser.add_argument(\n '--summaries_dir',\n type=str,\n default='/tmp/retrain_logs',\n help='Where to save summary logs for TensorBoard.'\n )\n parser.add_argument(\n '--how_many_training_steps',\n type=int,\n default=4000,\n help='How many training steps to run before ending.'\n )\n parser.add_argument(\n '--learning_rate',\n type=float,\n default=0.01,\n help='How large a learning rate to use when training.'\n )\n parser.add_argument(\n '--testing_percentage',\n type=int,\n default=10,\n help='What percentage of images to use as a test set.'\n )\n parser.add_argument(\n '--validation_percentage',\n type=int,\n default=10,\n help='What percentage of images to use as a validation set.'\n )\n parser.add_argument(\n '--eval_step_interval',\n type=int,\n default=10,\n help='How often to evaluate the training results.'\n )\n parser.add_argument(\n '--train_batch_size',\n type=int,\n default=100,\n help='How many images to train on at a time.'\n )\n parser.add_argument(\n '--test_batch_size',\n type=int,\n default=-1,\n help=\"\"\"\\\n How many images to test on. This test set is only used once, to evaluate\n the final accuracy of the model after training completes.\n A value of -1 causes the entire test set to be used, which leads to more\n stable results across runs.\\\n \"\"\"\n )\n parser.add_argument(\n '--validation_batch_size',\n type=int,\n default=100,\n help=\"\"\"\\\n How many images to use in an evaluation batch. This validation set is\n used much more often than the test set, and is an early indicator of how\n accurate the model is during training.\n A value of -1 causes the entire validation set to be used, which leads to\n more stable results across training iterations, but may be slower on large\n training sets.\\\n \"\"\"\n )\n parser.add_argument(\n '--print_misclassified_test_images',\n default=False,\n help=\"\"\"\\\n Whether to print out a list of all misclassified test images.\\\n \"\"\",\n action='store_true'\n )\n parser.add_argument(\n '--model_dir',\n type=str,\n default='/tmp/imagenet',\n help=\"\"\"\\\n Path to classify_image_graph_def.pb,\n imagenet_synset_to_human_label_map.txt, and\n imagenet_2012_challenge_label_map_proto.pbtxt.\\\n \"\"\"\n )\n parser.add_argument(\n '--bottleneck_dir',\n type=str,\n default='/tmp/bottleneck',\n help='Path to cache bottleneck layer values as files.'\n )\n parser.add_argument(\n '--final_tensor_name',\n type=str,\n default='final_result',\n help=\"\"\"\\\n The name of the output classification layer in the retrained graph.\\\n \"\"\"\n )\n parser.add_argument(\n '--flip_left_right',\n default=False,\n help=\"\"\"\\\n Whether to randomly flip half of the training images horizontally.\\\n \"\"\",\n action='store_true'\n )\n parser.add_argument(\n '--random_crop',\n type=int,\n default=0,\n help=\"\"\"\\\n A percentage determining how much of a margin to randomly crop off the\n training images.\\\n \"\"\"\n )\n parser.add_argument(\n '--random_scale',\n type=int,\n default=0,\n help=\"\"\"\\\n A percentage determining how much to randomly scale up the size of the\n training images by.\\\n \"\"\"\n )\n parser.add_argument(\n '--random_brightness',\n type=int,\n default=0,\n help=\"\"\"\\\n A percentage determining how much to randomly multiply the training image\n input pixels up or down by.\\\n \"\"\"\n )\n parser.add_argument(\n '--architecture',\n type=str,\n default='inception_v3',\n help=\"\"\"\\\n Which model architecture to use. 'inception_v3' is the most accurate, but\n also the slowest. For faster or smaller models, chose a MobileNet with the\n form 'mobilenet__[_quantized]'. For example,\n 'mobilenet_1.0_224' will pick a model that is 17 MB in size and takes 224\n pixel input images, while 'mobilenet_0.25_128_quantized' will choose a much\n less accurate, but smaller and faster network that's 920 KB on disk and\n takes 128x128 images. See https://research.googleblog.com/2017/06/mobilenets-open-source-models-for.html\n for more information on Mobilenet.\\\n \"\"\")\n FLAGS, unparsed = parser.parse_known_args()\n tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)\n" }, { "path": "setup_cifar.py", "content": "## setup_cifar.py -- cifar data and model loading code\n##\n## Copyright (C) IBM Corp, 2017-2018\n## Copyright (C) 2016, Nicholas Carlini .\n##\n## This program is licenced under the BSD 2-Clause licence,\n## contained in the LICENCE file in this directory.\n\n\nimport tensorflow as tf\nimport numpy as np\nimport os\nimport pickle\nimport gzip\nimport pickle\nimport urllib.request\n\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Activation, Flatten\nfrom keras.layers import Conv2D, MaxPooling2D\nfrom keras.utils import np_utils\nfrom keras.models import load_model\n\ndef load_batch(fpath, label_key='labels'):\n f = open(fpath, 'rb')\n d = pickle.load(f, encoding=\"bytes\")\n for k, v in d.items():\n del(d[k])\n d[k.decode(\"utf8\")] = v\n f.close()\n data = d[\"data\"]\n labels = d[label_key]\n\n data = data.reshape(data.shape[0], 3, 32, 32)\n final = np.zeros((data.shape[0], 32, 32, 3),dtype=np.float32)\n final[:,:,:,0] = data[:,0,:,:]\n final[:,:,:,1] = data[:,1,:,:]\n final[:,:,:,2] = data[:,2,:,:]\n\n final /= 255\n final -= .5\n labels2 = np.zeros((len(labels), 10))\n labels2[np.arange(len(labels2)), labels] = 1\n\n return final, labels\n\ndef load_batch(fpath):\n f = open(fpath,\"rb\").read()\n size = 32*32*3+1\n labels = []\n images = []\n for i in range(10000):\n arr = np.fromstring(f[i*size:(i+1)*size],dtype=np.uint8)\n lab = np.identity(10)[arr[0]]\n img = arr[1:].reshape((3,32,32)).transpose((1,2,0))\n\n labels.append(lab)\n images.append((img/255)-.5)\n return np.array(images),np.array(labels)\n \n\nclass CIFAR:\n def __init__(self):\n train_data = []\n train_labels = []\n \n if not os.path.exists(\"cifar-10-batches-bin\"):\n urllib.request.urlretrieve(\"https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz\",\n \"cifar-data.tar.gz\")\n os.popen(\"tar -xzf cifar-data.tar.gz\").read()\n \n\n for i in range(5):\n r,s = load_batch(\"cifar-10-batches-bin/data_batch_\"+str(i+1)+\".bin\")\n train_data.extend(r)\n train_labels.extend(s)\n \n train_data = np.array(train_data,dtype=np.float32)\n train_labels = np.array(train_labels)\n \n self.test_data, self.test_labels = load_batch(\"cifar-10-batches-bin/test_batch.bin\")\n \n VALIDATION_SIZE = 5000\n \n self.validation_data = train_data[:VALIDATION_SIZE, :, :, :]\n self.validation_labels = train_labels[:VALIDATION_SIZE]\n self.train_data = train_data[VALIDATION_SIZE:, :, :, :]\n self.train_labels = train_labels[VALIDATION_SIZE:]\n\nclass CIFARModel:\n def __init__(self, restore=None, session=None, use_log=False):\n self.num_channels = 3\n self.image_size = 32\n self.num_labels = 10\n\n model = Sequential()\n\n model.add(Conv2D(64, (3, 3),\n input_shape=(32, 32, 3)))\n model.add(Activation('relu'))\n model.add(Conv2D(64, (3, 3)))\n model.add(Activation('relu'))\n model.add(MaxPooling2D(pool_size=(2, 2)))\n \n model.add(Conv2D(128, (3, 3)))\n model.add(Activation('relu'))\n model.add(Conv2D(128, (3, 3)))\n model.add(Activation('relu'))\n model.add(MaxPooling2D(pool_size=(2, 2)))\n \n model.add(Flatten())\n model.add(Dense(256))\n model.add(Activation('relu'))\n model.add(Dense(256))\n model.add(Activation('relu'))\n model.add(Dense(10))\n if use_log:\n model.add(Activation('softmax'))\n if restore:\n model.load_weights(restore)\n\n self.model = model\n\n def predict(self, data):\n return self.model(data)\n \n \n" }, { "path": "setup_inception.py", "content": "## Copyright (C) IBM Corp, 2017-2018\n## Modified by Huan Zhang for the updated Inception-v3 model (inception_v3_2016_08_28.tar.gz)\n## Modified by Nicholas Carlini to match model structure for attack code.\n## Original copyright license follows.\n\n\n# Copyright 2015 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ==============================================================================\n\n\"\"\"Simple image classification with Inception.\n\nRun image classification with Inception trained on ImageNet 2012 Challenge data\nset.\n\nThis program creates a graph from a saved GraphDef protocol buffer,\nand runs inference on an input JPEG image. It outputs human readable\nstrings of the top 5 predictions along with their probabilities.\n\nChange the --image_file argument to any jpg image to compute a\nclassification of that image.\n\nPlease see the tutorial and website for a detailed description of how\nto use this script to perform image recognition.\n\nhttps://tensorflow.org/tutorials/image_recognition/\n\"\"\"\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport os.path\nimport re\nimport sys\nimport random\nimport tarfile\nimport scipy.misc\n\nimport numpy as np\nfrom six.moves import urllib\nimport tensorflow as tf\n\n# pylint: disable=line-too-long\nDATA_URL = 'http://download.huan-zhang.com/models/adv/imagenet/inception_v3_2016_08_28_frozen.tar.gz'\n# pylint: enable=line-too-long\n\n\nclass NodeLookup(object):\n \"\"\"Converts integer node ID's to human readable labels.\"\"\"\n\n def __init__(self,\n label_lookup_path=None):\n if not label_lookup_path:\n label_lookup_path = os.path.join(\n FLAGS.model_dir, 'labels.txt')\n self.node_lookup = self.load(label_lookup_path)\n\n def load(self, label_lookup_path):\n \"\"\"Loads a human readable English name for each softmax node.\n\n Args:\n label_lookup_path: string UID to integer node ID.\n uid_lookup_path: string UID to human-readable string.\n\n Returns:\n dict from integer node ID to human-readable string.\n \"\"\"\n if not tf.gfile.Exists(label_lookup_path):\n tf.logging.fatal('File does not exist %s', label_lookup_path)\n\n # Loads mapping from string UID to integer node ID.\n node_id_to_name = {}\n proto_as_ascii = tf.gfile.GFile(label_lookup_path).readlines()\n for line in proto_as_ascii:\n if line:\n words = line.split(':')\n target_class = int(words[0])\n name = words[1]\n node_id_to_name[target_class] = name\n\n return node_id_to_name\n\n def id_to_string(self, node_id):\n if node_id not in self.node_lookup:\n return ''\n return self.node_lookup[node_id]\n\n\ndef create_graph():\n \"\"\"Creates a graph from saved GraphDef file and returns a saver.\"\"\"\n # Creates graph from saved graph_def.pb.\n with tf.gfile.FastGFile(os.path.join(\n # FLAGS.model_dir, 'classify_image_graph_def.pb'), 'rb') as f:\n FLAGS.model_dir, 'frozen_inception_v3.pb'), 'rb') as f:\n graph_def = tf.GraphDef()\n graph_def.ParseFromString(f.read())\n #for line in repr(graph_def).split(\"\\n\"):\n # if \"tensor_content\" not in line:\n # print(line)\n _ = tf.import_graph_def(graph_def, name='')\n\n\ndef run_inference_on_image(image):\n \"\"\"Runs inference on an image. (Not updated, not working for inception v3 20160828)\n\n Args:\n image: Image file name.\n\n Returns:\n Nothing\n \"\"\"\n if not tf.gfile.Exists(image):\n tf.logging.fatal('File does not exist %s', image)\n image_data = tf.gfile.FastGFile(image, 'rb').read()\n\n # Creates graph from saved GraphDef.\n create_graph()\n\n with tf.Session() as sess:\n # Some useful tensors:\n # 'softmax:0': A tensor containing the normalized prediction across\n # 1000 labels.\n # 'pool_3:0': A tensor containing the next-to-last layer containing 2048\n # float description of the image.\n # 'DecodeJpeg/contents:0': A tensor containing a string providing JPEG\n # encoding of the image.\n # Runs the softmax tensor by feeding the image_data as input to the graph.\n #softmax_tensor = sess.graph.get_tensor_by_name('softmax:0')\n img = tf.placeholder(tf.uint8, (299,299,3))\n softmax_tensor = tf.import_graph_def(\n sess.graph.as_graph_def(),\n input_map={'DecodeJpeg:0': tf.reshape(img,((299,299,3)))},\n return_elements=['softmax/logits:0'])\n\n dat = scipy.misc.imresize(scipy.misc.imread(image),(299,299))\n predictions = sess.run(softmax_tensor,\n {img: dat})\n\n predictions = np.squeeze(predictions)\n\n # Creates node ID --> English string lookup.\n node_lookup = NodeLookup()\n\n top_k = predictions.argsort()#[-FLAGS.num_top_predictions:][::-1]\n for node_id in top_k:\n print('id',node_id)\n human_string = node_lookup.id_to_string(node_id)\n score = predictions[node_id]\n print('%s (score = %.5f)' % (human_string, score))\n\nclass InceptionModelPrediction:\n def __init__(self, sess, use_log = False):\n self.sess = sess\n self.use_log = use_log\n if self.use_log:\n output_name = 'InceptionV3/Predictions/Softmax:0'\n else:\n output_name = 'InceptionV3/Predictions/Reshape:0'\n self.img = tf.placeholder(tf.float32, (None, 299,299,3))\n self.softmax_tensor = tf.import_graph_def(\n sess.graph.as_graph_def(),\n input_map={'input:0': self.img},\n return_elements=[output_name])\n def predict(self, dat):\n dat = np.squeeze(dat)\n # scaled = (0.5 + dat) * 255\n scaled = dat.reshape((1,) + dat.shape)\n # print(scaled.shape)\n predictions = self.sess.run(self.softmax_tensor,\n {self.img: scaled})\n predictions = np.squeeze(predictions)\n return predictions\n # Creates node ID --> English string lookup.\n node_lookup = NodeLookup()\n top_k = predictions.argsort()#[-FLAGS.num_top_predictions:][::-1]\n for node_id in top_k:\n print('id',node_id)\n human_string = node_lookup.id_to_string(node_id)\n score = predictions[node_id]\n print('%s (score = %.5f)' % (human_string, score))\n return top_k[-1]\n\n\nCREATED_GRAPH = False\nclass InceptionModel:\n image_size = 299\n num_labels = 1001\n num_channels = 3\n def __init__(self, sess, use_log = False):\n global CREATED_GRAPH\n self.sess = sess\n self.use_log = use_log\n if not CREATED_GRAPH:\n create_graph()\n CREATED_GRAPH = True\n self.model = InceptionModelPrediction(sess, use_log)\n\n def predict(self, img):\n if self.use_log:\n output_name = 'InceptionV3/Predictions/Softmax:0'\n else:\n output_name = 'InceptionV3/Predictions/Reshape:0'\n # scaled = (0.5+tf.reshape(img,((299,299,3))))*255\n # scaled = (0.5+img)*255\n if img.shape.as_list()[0]:\n # check if a shape has been specified explicitly\n shape = (int(img.shape[0]), 1001)\n softmax_tensor = tf.import_graph_def(\n self.sess.graph.as_graph_def(),\n input_map={'input:0': img, 'InceptionV3/Predictions/Shape:0': shape},\n return_elements=[output_name])\n else:\n # placeholder shape\n softmax_tensor = tf.import_graph_def(\n self.sess.graph.as_graph_def(),\n input_map={'input:0': img},\n return_elements=[output_name])\n return softmax_tensor[0]\n \n\ndef maybe_download_and_extract():\n \"\"\"Download and extract model tar file.\"\"\"\n dest_directory = FLAGS.model_dir\n if not os.path.exists(dest_directory):\n os.makedirs(dest_directory)\n filename = DATA_URL.split('/')[-1]\n filepath = os.path.join(dest_directory, filename)\n if not os.path.exists(filepath):\n def _progress(count, block_size, total_size):\n sys.stdout.write('\\r>> Downloading %s %.1f%%' % (\n filename, float(count * block_size) / float(total_size) * 100.0))\n sys.stdout.flush()\n filepath, _ = urllib.request.urlretrieve(DATA_URL, filepath, _progress)\n print()\n statinfo = os.stat(filepath)\n print('Succesfully downloaded', filename, statinfo.st_size, 'bytes.')\n tarfile.open(filepath, 'r:gz').extractall(dest_directory)\n\n\ndef main(_):\n maybe_download_and_extract()\n image = (FLAGS.image_file if FLAGS.image_file else\n os.path.join(FLAGS.model_dir, 'cropped_panda.jpg'))\n # run_inference_on_image(image)\n create_graph()\n with tf.Session() as sess:\n dat = np.array(scipy.misc.imresize(scipy.misc.imread(image),(299,299)), dtype = np.float32)\n dat /= 255.0\n dat -= 0.5\n # print(dat)\n model = InceptionModelPrediction(sess, True)\n predictions = model.predict(dat)\n # Creates node ID --> English string lookup.\n node_lookup = NodeLookup()\n top_k = predictions.argsort()#[-FLAGS.num_top_predictions:][::-1]\n for node_id in top_k:\n print('id',node_id)\n human_string = node_lookup.id_to_string(node_id)\n score = predictions[node_id]\n print('%s (score = %.5f)' % (human_string, score))\n\n\ndef readimg(ff):\n f = \"../imagenetdata/imgs/\"+ff\n img = scipy.misc.imread(f)\n # skip small images (image should be at least 299x299)\n if img.shape[0] < 299 or img.shape[1] < 299:\n return None\n img = np.array(scipy.misc.imresize(img,(299,299)),dtype=np.float32)/255-.5\n if img.shape != (299, 299, 3):\n return None\n return [img, int(ff.split(\".\")[0])]\n\nclass ImageNet:\n def __init__(self):\n from multiprocessing import Pool\n pool = Pool(8)\n file_list = sorted(os.listdir(\"../imagenetdata/imgs/\"))\n random.shuffle(file_list)\n r = pool.map(readimg, file_list[:200])\n print(file_list[:200])\n r = [x for x in r if x != None]\n test_data, test_labels = zip(*r)\n self.test_data = np.array(test_data)\n self.test_labels = np.zeros((len(test_labels), 1001))\n self.test_labels[np.arange(len(test_labels)), test_labels] = 1\n\n \n\n\nif __name__ == '__main__':\n FLAGS = tf.app.flags.FLAGS\n # classify_image_graph_def.pb:\n # Binary representation of the GraphDef protocol buffer.\n # imagenet_synset_to_human_label_map.txt:\n # Map from synset ID to a human readable string.\n # imagenet_2012_challenge_label_map_proto.pbtxt:\n # Text representation of a protocol buffer mapping a label to synset ID.\n tf.app.flags.DEFINE_string(\n 'model_dir', 'tmp/imagenet',\n \"\"\"Path to classify_image_graph_def.pb, \"\"\"\n \"\"\"imagenet_synset_to_human_label_map.txt, and \"\"\"\n \"\"\"imagenet_2012_challenge_label_map_proto.pbtxt.\"\"\")\n tf.app.flags.DEFINE_string('image_file', '',\n\t\t\t \"\"\"Absolute path to image file.\"\"\")\n tf.app.flags.DEFINE_integer('num_top_predictions', 5,\n\t\t\t \"\"\"Display this many predictions.\"\"\")\n tf.app.run()\nelse:\n from argparse import Namespace\n FLAGS = Namespace(model_dir=\"tmp/imagenet\") \n" }, { "path": "setup_mnist.py", "content": "## setup_mnist.py -- mnist data and model loading code\n##\n## Copyright (C) IBM Corp, 2017-2018\n## Copyright (C) 2016, Nicholas Carlini .\n##\n## This program is licenced under the BSD 2-Clause licence,\n## contained in the LICENCE file in this directory.\n\nimport tensorflow as tf\nimport numpy as np\nimport os\nimport pickle\nimport gzip\nimport urllib.request\n\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Activation, Flatten\nfrom keras.layers import Conv2D, MaxPooling2D\nfrom keras.utils import np_utils\nfrom keras.models import load_model\n\ndef extract_data(filename, num_images):\n with gzip.open(filename) as bytestream:\n bytestream.read(16)\n buf = bytestream.read(num_images*28*28)\n data = np.frombuffer(buf, dtype=np.uint8).astype(np.float32)\n data = (data / 255) - 0.5\n data = data.reshape(num_images, 28, 28, 1)\n return data\n\ndef extract_labels(filename, num_images):\n with gzip.open(filename) as bytestream:\n bytestream.read(8)\n buf = bytestream.read(1 * num_images)\n labels = np.frombuffer(buf, dtype=np.uint8)\n return (np.arange(10) == labels[:, None]).astype(np.float32)\n\nclass MNIST:\n def __init__(self):\n if not os.path.exists(\"data\"):\n os.mkdir(\"data\")\n files = [\"train-images-idx3-ubyte.gz\",\n \"t10k-images-idx3-ubyte.gz\",\n \"train-labels-idx1-ubyte.gz\",\n \"t10k-labels-idx1-ubyte.gz\"]\n for name in files:\n\n urllib.request.urlretrieve('http://yann.lecun.com/exdb/mnist/' + name, \"data/\"+name)\n\n train_data = extract_data(\"data/train-images-idx3-ubyte.gz\", 60000)\n train_labels = extract_labels(\"data/train-labels-idx1-ubyte.gz\", 60000)\n self.test_data = extract_data(\"data/t10k-images-idx3-ubyte.gz\", 10000)\n self.test_labels = extract_labels(\"data/t10k-labels-idx1-ubyte.gz\", 10000)\n \n VALIDATION_SIZE = 5000\n \n self.validation_data = train_data[:VALIDATION_SIZE, :, :, :]\n self.validation_labels = train_labels[:VALIDATION_SIZE]\n self.train_data = train_data[VALIDATION_SIZE:, :, :, :]\n self.train_labels = train_labels[VALIDATION_SIZE:]\n\n\nclass MNISTModel:\n def __init__(self, restore = None, session=None, use_log=False):\n self.num_channels = 1\n self.image_size = 28\n self.num_labels = 10\n\n model = Sequential()\n\n model.add(Conv2D(32, (3, 3),\n input_shape=(28, 28, 1)))\n model.add(Activation('relu'))\n model.add(Conv2D(32, (3, 3)))\n model.add(Activation('relu'))\n model.add(MaxPooling2D(pool_size=(2, 2)))\n \n model.add(Conv2D(64, (3, 3)))\n model.add(Activation('relu'))\n model.add(Conv2D(64, (3, 3)))\n model.add(Activation('relu'))\n model.add(MaxPooling2D(pool_size=(2, 2)))\n \n model.add(Flatten())\n model.add(Dense(200))\n model.add(Activation('relu'))\n model.add(Dense(200))\n model.add(Activation('relu'))\n model.add(Dense(10))\n # output log probability, used for black-box attack\n if use_log:\n model.add(Activation('softmax'))\n if restore:\n model.load_weights(restore)\n\n self.model = model\n\n def predict(self, data):\n return self.model(data)\n\n" }, { "path": "substitute_blackbox.py", "content": "## Copyright (C) IBM Corp, 2017-2018\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\n\nimport os\nimport time\nimport numpy as np\nfrom six.moves import xrange\n\nimport keras\nfrom keras import backend\nfrom keras.utils.np_utils import to_categorical\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Flatten, Activation, Dropout\nfrom keras.datasets import cifar10\nfrom keras.utils import np_utils\n\nimport tensorflow as tf\nfrom tensorflow.python.platform import app\nfrom tensorflow.python.platform import flags\n\nfrom cleverhans.utils_keras import cnn_model\nfrom cleverhans.utils_mnist import data_mnist\nfrom cleverhans.utils_tf import model_train, model_eval, batch_eval, tf_model_load\nfrom cleverhans.attacks import CarliniWagnerL2\nfrom cleverhans.attacks import FastGradientMethod\nfrom cleverhans.attacks_tf import jacobian_graph, jacobian_augmentation\nfrom cleverhans.utils_keras import KerasModelWrapper\n\nfrom setup_mnist import MNISTModel\nfrom setup_cifar import CIFARModel\n\nFLAGS = flags.FLAGS\n\nDATASET = \"cifar\"\n\ndef data_cifar10():\n \"\"\"\n Preprocess CIFAR10 dataset\n :return:\n \"\"\"\n\n # These values are specific to CIFAR10\n img_rows = 32\n img_cols = 32\n nb_classes = 10\n\n # the data, shuffled and split between train and test sets\n (X_train, y_train), (X_test, y_test) = cifar10.load_data()\n\n if keras.backend.image_dim_ordering() == 'th':\n X_train = X_train.reshape(X_train.shape[0], 3, img_rows, img_cols)\n X_test = X_test.reshape(X_test.shape[0], 3, img_rows, img_cols)\n else:\n X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 3)\n X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 3)\n X_train = X_train.astype('float32')\n X_test = X_test.astype('float32')\n X_train /= 255\n X_test /= 255\n print('X_train shape:', X_train.shape)\n print(X_train.shape[0], 'train samples')\n print(X_test.shape[0], 'test samples')\n\n # convert class vectors to binary class matrices\n Y_train = np_utils.to_categorical(y_train, nb_classes)\n Y_test = np_utils.to_categorical(y_test, nb_classes)\n return X_train, Y_train, X_test, Y_test\n\ndef setup_tutorial():\n \"\"\"\n Helper function to check correct configuration of tf and keras for tutorial\n :return: True if setup checks completed\n \"\"\"\n\n # Set TF random seed to improve reproducibility\n tf.set_random_seed(1234)\n\n if not hasattr(backend, \"tf\"):\n raise RuntimeError(\"This tutorial requires keras to be configured\"\n \" to use the TensorFlow backend.\")\n\n # Image dimensions ordering should follow the Theano convention\n if keras.backend.image_dim_ordering() != 'tf':\n keras.backend.set_image_dim_ordering('tf')\n print(\"INFO: '~/.keras/keras.json' sets 'image_dim_ordering' \"\n \"to 'th', temporarily setting to 'tf'\")\n\n return True\n\n\ndef prep_bbox(sess, x, y, X_train, Y_train, X_test, Y_test,\n nb_epochs, batch_size, learning_rate):\n \"\"\"\n Define and train a model that simulates the \"remote\"\n black-box oracle described in the original paper.\n :param sess: the TF session\n :param x: the input placeholder for MNIST\n :param y: the ouput placeholder for MNIST\n :param X_train: the training data for the oracle\n :param Y_train: the training labels for the oracle\n :param X_test: the testing data for the oracle\n :param Y_test: the testing labels for the oracle\n :param nb_epochs: number of epochs to train model\n :param batch_size: size of training batches\n :param learning_rate: learning rate for training\n :return:\n \"\"\"\n\n # Define TF model graph (for the black-box model)\n if DATASET == \"mnist\":\n model = MNISTModel(use_log = True).model\n else:\n model = CIFARModel(use_log = True).model\n predictions = model(x)\n print(\"Defined TensorFlow model graph.\")\n\n # Train an MNIST model\n if FLAGS.load_pretrain:\n tf_model_load(sess)\n else:\n train_params = {\n 'nb_epochs': nb_epochs,\n 'batch_size': batch_size,\n 'learning_rate': learning_rate\n }\n model_train(sess, x, y, predictions, X_train, Y_train, verbose=True, save=True,\n args=train_params)\n\n # Print out the accuracy on legitimate data\n eval_params = {'batch_size': batch_size}\n accuracy = model_eval(sess, x, y, predictions, X_test, Y_test,\n args=eval_params)\n print('Test accuracy of black-box on legitimate test '\n 'examples: ' + str(accuracy))\n\n return model, predictions, accuracy\n\n\ndef substitute_model(img_rows=28, img_cols=28, nb_classes=10):\n \"\"\"\n Defines the model architecture to be used by the substitute\n :param img_rows: number of rows in input\n :param img_cols: number of columns in input\n :param nb_classes: number of classes in output\n :return: keras model\n \"\"\"\n model = Sequential()\n\n # Find out the input shape ordering\n if keras.backend.image_dim_ordering() == 'th':\n input_shape = (1, img_rows, img_cols)\n else:\n input_shape = (img_rows, img_cols, 1)\n\n # Define a fully connected model (it's different than the black-box)\n layers = [Flatten(input_shape=input_shape),\n Dense(200),\n Activation('relu'),\n Dropout(0.5),\n Dense(200),\n Activation('relu'),\n Dropout(0.5),\n Dense(nb_classes),\n Activation('softmax')]\n\n for layer in layers:\n model.add(layer)\n\n return model\n\n\ndef train_sub(sess, x, y, bbox_preds, X_sub, Y_sub, nb_classes,\n nb_epochs_s, batch_size, learning_rate, data_aug, lmbda):\n \"\"\"\n This function creates the substitute by alternatively\n augmenting the training data and training the substitute.\n :param sess: TF session\n :param x: input TF placeholder\n :param y: output TF placeholder\n :param bbox_preds: output of black-box model predictions\n :param X_sub: initial substitute training data\n :param Y_sub: initial substitute training labels\n :param nb_classes: number of output classes\n :param nb_epochs_s: number of epochs to train substitute model\n :param batch_size: size of training batches\n :param learning_rate: learning rate for training\n :param data_aug: number of times substitute training data is augmented\n :param lmbda: lambda from arxiv.org/abs/1602.02697\n :return:\n \"\"\"\n # Define TF model graph (for the black-box model)\n # model_sub = substitute_model()\n if DATASET == \"mnist\":\n model_sub = MNISTModel(use_log = True).model\n else:\n model_sub = CIFARModel(use_log = True).model\n preds_sub = model_sub(x)\n print(\"Defined TensorFlow model graph for the substitute.\")\n\n # Define the Jacobian symbolically using TensorFlow\n grads = jacobian_graph(preds_sub, x, nb_classes)\n\n # Train the substitute and augment dataset alternatively\n for rho in xrange(data_aug):\n print(\"Substitute training epoch #\" + str(rho))\n train_params = {\n 'nb_epochs': nb_epochs_s,\n 'batch_size': batch_size,\n 'learning_rate': learning_rate\n }\n model_train(sess, x, y, preds_sub, X_sub, to_categorical(Y_sub),\n init_all=False, verbose=False, args=train_params)\n\n # If we are not at last substitute training iteration, augment dataset\n if rho < data_aug - 1:\n if FLAGS.cached_aug:\n augs = np.load('sub_saved/{}-aug-{}.npz'.format(DATASET, rho))\n X_sub = augs['X_sub']\n Y_sub = augs['Y_sub']\n else:\n print(\"Augmenting substitute training data.\")\n # Perform the Jacobian augmentation\n X_sub = jacobian_augmentation(sess, x, X_sub, Y_sub, grads, lmbda)\n\n print(\"Labeling substitute training data.\")\n # Label the newly generated synthetic points using the black-box\n Y_sub = np.hstack([Y_sub, Y_sub])\n X_sub_prev = X_sub[int(len(X_sub)/2):]\n eval_params = {'batch_size': batch_size}\n bbox_val = batch_eval(sess, [x], [bbox_preds], [X_sub_prev],\n args=eval_params)[0]\n # Note here that we take the argmax because the adversary\n # only has access to the label (not the probabilities) output\n # by the black-box model\n Y_sub[int(len(X_sub)/2):] = np.argmax(bbox_val, axis=1)\n # cache the augmentation\n if not FLAGS.cached_aug:\n np.savez('sub_saved/{}-aug-{}.npz'.format(DATASET, rho), X_sub = X_sub, Y_sub = Y_sub)\n\n return model_sub, preds_sub\n\n\ndef mnist_blackbox(train_start=0, train_end=60000, test_start=0,\n test_end=10000, nb_classes=10, batch_size=128,\n learning_rate=0.001, nb_epochs=10, holdout=150, data_aug=6,\n nb_epochs_s=10, lmbda=0.1, attack=\"fgsm\", targeted=False):\n \"\"\"\n MNIST tutorial for the black-box attack from arxiv.org/abs/1602.02697\n :param train_start: index of first training set example\n :param train_end: index of last training set example\n :param test_start: index of first test set example\n :param test_end: index of last test set example\n :return: a dictionary with:\n * black-box model accuracy on test set\n * substitute model accuracy on test set\n * black-box model accuracy on adversarial examples transferred\n from the substitute model\n \"\"\"\n keras.layers.core.K.set_learning_phase(0)\n\n # Dictionary used to keep track and return key accuracies\n accuracies = {}\n\n # Perform tutorial setup\n assert setup_tutorial()\n\n # Create TF session and set as Keras backend session\n gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=1.0)\n sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))\n keras.backend.set_session(sess)\n\n # Get MNIST data\n if DATASET == \"mnist\":\n X_train, Y_train, X_test, Y_test = data_mnist(train_start=train_start,\n train_end=train_end,\n test_start=test_start,\n test_end=test_end)\n else:\n X_train, Y_train, X_test, Y_test = data_cifar10()\n\n # Initialize substitute training set reserved for adversary\n X_sub = X_test[:holdout]\n Y_sub = np.argmax(Y_test[:holdout], axis=1)\n\n # Redefine test set as remaining samples unavailable to adversaries\n X_test = X_test[holdout:]\n Y_test = Y_test[holdout:]\n\n X_test = X_test[:FLAGS.n_attack]\n Y_test = Y_test[:FLAGS.n_attack]\n\n # Define input and output TF placeholders\n if DATASET == \"mnist\":\n x = tf.placeholder(tf.float32, shape=(None, 28, 28, 1))\n else:\n x = tf.placeholder(tf.float32, shape=(None, 32, 32, 3))\n y = tf.placeholder(tf.float32, shape=(None, 10))\n # for feed targeted attack labels\n t_y = tf.placeholder(tf.float32, shape=(None, 10))\n\n # Simulate the black-box model locally\n # You could replace this by a remote labeling API for instance\n print(\"Preparing the black-box model.\")\n prep_bbox_out = prep_bbox(sess, x, y, X_train, Y_train, X_test, Y_test,\n nb_epochs, batch_size, learning_rate)\n model, bbox_preds, accuracies['bbox'] = prep_bbox_out\n\n # Train substitute using method from https://arxiv.org/abs/1602.02697\n time_start = time.time()\n print(\"Training the substitute model.\")\n train_sub_out = train_sub(sess, x, y, bbox_preds, X_sub, Y_sub,\n nb_classes, nb_epochs_s, batch_size,\n learning_rate, data_aug, lmbda)\n model_sub, preds_sub = train_sub_out\n time_end = time.time()\n print(\"Substitue model training time:\", time_end - time_start)\n\n # Evaluate the substitute model on clean test examples\n eval_params = {'batch_size': batch_size}\n acc = model_eval(sess, x, y, preds_sub, X_test, Y_test, args=eval_params)\n accuracies['sub'] = acc\n print('substitution model accuracy:', acc)\n\n # Find the correctly predicted labels\n original_predict = batch_eval(sess, [x], [bbox_preds], [X_test],\n args=eval_params)[0]\n original_class = np.argmax(original_predict, axis = 1)\n true_class = np.argmax(Y_test, axis = 1)\n mask = true_class == original_class\n print(np.sum(mask), \"out of\", mask.size, \"are correct labeled,\", len(X_test[mask])) \n\n # Initialize the Fast Gradient Sign Method (FGSM) attack object.\n wrap = KerasModelWrapper(model_sub)\n\n\n # Craft adversarial examples using the substitute\n if targeted and attack == \"fgsm\":\n # TODO: fix the batch size mess\n eval_params = {'batch_size': FLAGS.n_attack * 9}\n else:\n eval_params = {'batch_size': batch_size}\n\n adv_inputs = X_test\n ori_labels = Y_test\n\n # generate targeted labels, 9 for each test example\n if targeted:\n adv_ys = []\n targeted_class = []\n for i in range(0, X_test.shape[0]):\n for j in range(0,10):\n # skip the original image label\n if j == np.argmax(Y_test[i]):\n continue\n adv_ys.append(np.eye(10)[j])\n targeted_class.append(j)\n # duplicate the inputs by 9 times\n adv_inputs = np.array([[instance] * 9 for instance in X_test],\n dtype=np.float32)\n if DATASET == \"mnist\":\n adv_inputs = adv_inputs.reshape((X_test.shape[0] * 9, 28, 28, 1))\n else:\n adv_inputs = adv_inputs.reshape((X_test.shape[0] * 9, 32, 32, 3))\n # also update the mask\n mask = np.repeat(mask, 9)\n ori_labels = np.repeat(Y_test, 9, axis=0)\n adv_ys = np.array(adv_ys, dtype=np.float32)\n \n if attack == \"fgsm\":\n attacker_params = {'eps': 0.4, 'ord': np.inf, 'clip_min': 0., 'clip_max': 1.}\n # wrap = KerasModelWrapper(model)\n fgsm = FastGradientMethod(wrap, sess=sess)\n attacker = fgsm\n print(\"Running FGSM attack...\")\n if targeted:\n attacker_params['y_target'] = t_y\n x_adv_sub = fgsm.generate(x, **attacker_params)\n else:\n print(\"Running Carlini and Wagner\\'s L2 attack...\")\n yname = \"y\"\n adv_ys = None\n # wrap = KerasModelWrapper(model)\n cwl2 = CarliniWagnerL2(wrap, back='tf', sess=sess)\n attacker_params = {'binary_search_steps': 9,\n 'max_iterations': 2000,\n 'abort_early': True,\n 'learning_rate': 0.01,\n 'batch_size': 1,\n 'initial_const': 0.01,\n 'confidence': 20}\n # generate targeted labels, 9 for each test example\n if targeted:\n attacker_params['y_target'] = adv_ys\n # attacker_params['batch_size'] = 9\n attacker = cwl2\n\n time_start = time.time()\n if attack == \"fgsm\":\n # Evaluate the accuracy of the \"black-box\" model on adversarial examples\n if targeted:\n accuracy = model_eval(sess, x, y, model(x_adv_sub), adv_inputs, ori_labels, feed={t_y: adv_ys},\n args=eval_params)\n else:\n accuracy = model_eval(sess, x, y, model(x_adv_sub), adv_inputs, ori_labels,\n args=eval_params)\n print('Test accuracy of oracle on adversarial examples generated '\n 'using the substitute: ' + str(accuracy))\n accuracies['bbox_on_sub_adv_ex'] = accuracy\n else:\n # Evaluate the accuracy of the \"black-box\" model on adversarial examples\n x_adv_sub_np = attacker.generate_np(adv_inputs, **attacker_params)\n accuracy = model_eval(sess, x, y, bbox_preds, x_adv_sub_np, ori_labels,\n args=eval_params)\n print('Test accuracy of oracle on adversarial examples generated '\n 'using the substitute (NP): ' + str(accuracy))\n accuracies['bbox_on_sub_adv_ex'] = accuracy\n time_end = time.time()\n print('Attack time:', time_end - time_start)\n\n # Evaluate the targeted attack\n if attack == \"fgsm\":\n bbox_adv_predict = batch_eval(sess, [x], [model(x_adv_sub)], [adv_inputs], feed={t_y: adv_ys},\n args=eval_params)[0]\n else:\n bbox_adv_predict = batch_eval(sess, [x], [bbox_preds], [x_adv_sub_np],\n args=eval_params)[0]\n bbox_adv_class = np.argmax(bbox_adv_predict, axis = 1)\n print(bbox_adv_class)\n print(true_class)\n true_class = np.argmax(ori_labels, axis = 1)\n untargeted_success = np.mean(bbox_adv_class != true_class)\n print('Untargeted attack success rate:', untargeted_success)\n accuracies['untargeted_success'] = untargeted_success\n if targeted:\n targeted_success = np.mean(bbox_adv_class == targeted_class)\n print('Targeted attack success rate:', targeted_success)\n accuracies['targeted_success'] = targeted_success\n\n if attack == \"cwl2\":\n # Compute the L2 pertubations of generated adversarial examples\n percent_perturbed = np.sum((x_adv_sub_np - adv_inputs)**2, axis=(1, 2, 3))**.5\n print(percent_perturbed)\n # print('Avg. L_2 norm of perturbations {0:.4f}'.format(np.mean(percent_perturbed)))\n # when computing the mean, removing the failure attacks first\n print('Avg. L_2 norm of all perturbations {0:.4f}'.format(np.mean(percent_perturbed[percent_perturbed > 1e-8])))\n print('Avg. L_2 norm of successful untargeted perturbations {0:.4f}'.format(np.mean(percent_perturbed[bbox_adv_class != true_class])))\n if targeted:\n print('Avg. L_2 norm of successful targeted perturbations {0:.4f}'.format(np.mean(percent_perturbed[bbox_adv_class == targeted_class])))\n\n # Evaluate the accuracy of the \"black-box\" model on adversarial examples\n accuracy = model_eval(sess, x, y, bbox_preds, adv_inputs[mask], ori_labels[mask],\n args=eval_params)\n print('Test accuracy of excluding originally incorrect labels (should be 1.0): ' + str(accuracy))\n accuracies['bbox_on_sub_adv_ex_exc_ori'] = accuracy\n\n if attack == \"fgsm\":\n # Evaluate the accuracy of the \"black-box\" model on adversarial examples (excluding correct)\n accuracy = model_eval(sess, x, y, model(x_adv_sub), adv_inputs[mask], ori_labels[mask], feed={t_y: adv_ys[mask]},\n args=eval_params)\n print('Test accuracy of oracle on adversarial examples generated '\n 'using the substitute (excluding originally incorrect labels): ' + str(accuracy))\n accuracies['bbox_on_sub_adv_ex_exc'] = accuracy\n else:\n # Evaluate the accuracy of the \"black-box\" model on adversarial examples (excluding correct)\n x_adv_sub_mask_np = x_adv_sub_np[mask]\n accuracy = model_eval(sess, x, y, bbox_preds, x_adv_sub_mask_np, ori_labels[mask],\n args=eval_params)\n print('Test accuracy of oracle on adversarial examples generated '\n 'using the substitute (excluding originally incorrect labels, NP): ' + str(accuracy))\n accuracies['bbox_on_sub_adv_ex_exc'] = accuracy\n\n return accuracies\n\n\ndef main(argv=None):\n print(\"DATASET:\", DATASET)\n print(\"Targeted:\", FLAGS.targeted)\n print(\"Attack:\", FLAGS.attack)\n print(\"Use Pretrained\", FLAGS.load_pretrain)\n print(\"Train Epochs:\", FLAGS.nb_epochs)\n print(\"Sub Train Epochs:\", FLAGS.nb_epochs_s)\n print(\"Holdout Size:\", FLAGS.holdout)\n print(\"Data Augmentation:\", FLAGS.data_aug)\n print(\"Number of Attacks:\", FLAGS.n_attack)\n mnist_blackbox(nb_classes=FLAGS.nb_classes, batch_size=FLAGS.batch_size,\n learning_rate=FLAGS.learning_rate,\n nb_epochs=FLAGS.nb_epochs, holdout=FLAGS.holdout,\n data_aug=FLAGS.data_aug, nb_epochs_s=FLAGS.nb_epochs_s,\n lmbda=FLAGS.lmbda, attack=FLAGS.attack, targeted=FLAGS.targeted)\n\n\nif __name__ == '__main__':\n # General flags\n flags.DEFINE_integer('nb_classes', 10, 'Number of classes in problem')\n flags.DEFINE_integer('batch_size', 128, 'Size of training batches')\n flags.DEFINE_integer('n_attack', -1, 'No. of images used for attack')\n if DATASET == \"mnist\":\n flags.DEFINE_float('learning_rate', 0.001, 'Learning rate for training')\n else:\n flags.DEFINE_float('learning_rate', 0.0005, 'Learning rate for training')\n\n # Flags related to oracle\n if DATASET == \"mnist\":\n flags.DEFINE_integer('nb_epochs', 10, 'Number of epochs to train model')\n else:\n flags.DEFINE_integer('nb_epochs', 50, 'Number of epochs to train model')\n\n # Flags related to substitute\n flags.DEFINE_integer('holdout', 150, 'Test set holdout for adversary')\n flags.DEFINE_integer('data_aug', 6, 'Nb of substitute data augmentations')\n if DATASET == \"mnist\":\n flags.DEFINE_integer('nb_epochs_s', 30, 'Training epochs for substitute')\n else:\n flags.DEFINE_integer('nb_epochs_s', 50, 'Training epochs for substitute')\n flags.DEFINE_float('lmbda', 0.1, 'Lambda from arxiv.org/abs/1602.02697')\n\n # Flags related to attack\n flags.DEFINE_string('attack', 'cwl2', 'cwl2 = Carlini & Wagner\\'s L2 attack, fgsm = Fast Gradient Sign Method')\n flags.DEFINE_bool('targeted', False, 'use targeted attack')\n\n # Flags related to saving/loading\n flags.DEFINE_bool('load_pretrain', False, 'load pretrained model from sub_saved/mnist-model')\n flags.DEFINE_bool('cached_aug', False, 'use cached augmentation in sub_saved')\n flags.DEFINE_string('train_dir', 'sub_saved', 'model saving path')\n if DATASET == \"mnist\":\n flags.DEFINE_string('filename', 'mnist-model', 'mnist model name')\n else:\n flags.DEFINE_string('filename', 'cifar-model', 'cifar model name')\n\n os.system(\"mkdir -p sub_saved\")\n\n app.run()\n" }, { "path": "test_all.py", "content": "## test_attack.py -- sample code to test attack procedure\n##\n## Copyright (C) IBM Corp, 2017-2018\n## Copyright (C) 2017, Huan Zhang .\n## Copyright (C) 2016, Nicholas Carlini .\n##\n## This program is licenced under the BSD 2-Clause licence,\n## contained in the LICENCE file in this directory.\n\nimport os\nimport sys\nimport tensorflow as tf\nimport numpy as np\nimport random\nimport time\n\nfrom setup_cifar import CIFAR, CIFARModel\nfrom setup_mnist import MNIST, MNISTModel\nfrom setup_inception import ImageNet, InceptionModel\n\nfrom l2_attack import CarliniL2\nfrom l2_attack_black import BlackBoxL2\n\nfrom PIL import Image\n\n\ndef show(img, name = \"output.png\"):\n \"\"\"\n Show MNSIT digits in the console.\n \"\"\"\n np.save(name, img)\n fig = np.around((img + 0.5)*255)\n fig = fig.astype(np.uint8).squeeze()\n pic = Image.fromarray(fig)\n # pic.resize((512,512), resample=PIL.Image.BICUBIC)\n pic.save(name)\n remap = \" .*#\"+\"#\"*100\n img = (img.flatten()+.5)*3\n if len(img) != 784: return\n print(\"START\")\n for i in range(28):\n print(\"\".join([remap[int(round(x))] for x in img[i*28:i*28+28]]))\n\ndef generate_data(data, samples, targeted=True, start=0, inception=False):\n \"\"\"\n Generate the input data to the attack algorithm.\n\n data: the images to attack\n samples: number of samples to use\n targeted: if true, construct targeted attacks, otherwise untargeted attacks\n start: offset into data to use\n inception: if targeted and inception, randomly sample 100 targets intead of 1000\n \"\"\"\n inputs = []\n targets = []\n labels = []\n true_ids = []\n for i in range(samples):\n if targeted:\n if inception:\n # for inception, randomly choose 10 target classes\n seq = np.random.choice(range(1,1001), 10)\n # seq = [580] # grand piano\n else:\n # for CIFAR and MNIST, generate all target classes\n seq = range(data.test_labels.shape[1])\n\n # print ('image label:', np.argmax(data.test_labels[start+i]))\n for j in seq:\n # skip the original image label\n if (j == np.argmax(data.test_labels[start+i])) and (inception == False):\n continue\n inputs.append(data.test_data[start+i])\n targets.append(np.eye(data.test_labels.shape[1])[j])\n labels.append(data.test_labels[start+i])\n true_ids.append(start+i)\n else:\n inputs.append(data.test_data[start+i])\n targets.append(data.test_labels[start+i])\n labels.append(data.test_labels[start+i])\n true_ids.append(start+i)\n\n inputs = np.array(inputs)\n targets = np.array(targets)\n labels = np.array(labels)\n true_ids = np.array(true_ids)\n\n return inputs, targets, labels, true_ids\n\ndef main(args):\n with tf.Session() as sess:\n use_log = not args['use_zvalue']\n is_inception = args['dataset'] == \"imagenet\"\n # load network\n print('Loading model', args['dataset'])\n if args['dataset'] == \"mnist\":\n data, model = MNIST(), MNISTModel(\"models/mnist\", sess, use_log)\n # data, model = MNIST(), MNISTModel(\"models/mnist-distilled-100\", sess, use_log)\n elif args['dataset'] == \"cifar10\":\n data, model = CIFAR(), CIFARModel(\"models/cifar\", sess, use_log)\n # data, model = CIFAR(), CIFARModel(\"models/cifar-distilled-100\", sess, use_log)\n elif args['dataset'] == \"imagenet\":\n data, model = ImageNet(), InceptionModel(sess, use_log)\n print('Done...')\n if args['numimg'] == 0:\n args['numimg'] = len(data.test_labels) - args['firstimg']\n print('Using', args['numimg'], 'test images')\n # load attack module\n if args['attack'] == \"white\":\n # batch size 1, optimize on 1 image at a time, rather than optimizing images jointly\n attack = CarliniL2(sess, model, batch_size=1, max_iterations=args['maxiter'], print_every=args['print_every'], \n early_stop_iters=args['early_stop_iters'], confidence=0, learning_rate = args['lr'], initial_const=args['init_const'], \n binary_search_steps=args['binary_steps'], targeted=not args['untargeted'], use_log=use_log,\n adam_beta1=args['adam_beta1'], adam_beta2=args['adam_beta2'])\n else:\n # batch size 128, optimize on 128 coordinates of a single image\n attack = BlackBoxL2(sess, model, batch_size=128, max_iterations=args['maxiter'], print_every=args['print_every'], \n early_stop_iters=args['early_stop_iters'], confidence=0, learning_rate = args['lr'], initial_const=args['init_const'], \n binary_search_steps=args['binary_steps'], targeted=not args['untargeted'], use_log=use_log, use_tanh=args['use_tanh'], \n use_resize=args['use_resize'], adam_beta1=args['adam_beta1'], adam_beta2=args['adam_beta2'], reset_adam_after_found=args['reset_adam'],\n solver=args['solver'], save_ckpts=args['save_ckpts'], load_checkpoint=args['load_ckpt'], start_iter=args['start_iter'],\n init_size=args['init_size'], use_importance=not args['uniform'])\n\n random.seed(args['seed'])\n np.random.seed(args['seed'])\n print('Generate data')\n all_inputs, all_targets, all_labels, all_true_ids = generate_data(data, samples=args['numimg'], targeted=not args['untargeted'],\n start=args['firstimg'], inception=is_inception)\n print('Done...')\n os.system(\"mkdir -p {}/{}\".format(args['save'], args['dataset']))\n img_no = 0\n total_success = 0\n l2_total = 0.0\n for i in range(all_true_ids.size):\n inputs = all_inputs[i:i+1]\n targets = all_targets[i:i+1]\n labels = all_labels[i:i+1]\n print(\"true labels:\", np.argmax(labels), labels)\n print(\"target:\", np.argmax(targets), targets)\n # test if the image is correctly classified\n original_predict = model.model.predict(inputs)\n original_predict = np.squeeze(original_predict)\n original_prob = np.sort(original_predict)\n original_class = np.argsort(original_predict)\n print(\"original probabilities:\", original_prob[-1:-6:-1])\n print(\"original classification:\", original_class[-1:-6:-1])\n print(\"original probabilities (most unlikely):\", original_prob[:6])\n print(\"original classification (most unlikely):\", original_class[:6])\n if original_class[-1] != np.argmax(labels):\n print(\"skip wrongly classified image no. {}, original class {}, classified as {}\".format(i, np.argmax(labels), original_class[-1]))\n continue\n \n img_no += 1\n timestart = time.time()\n adv, const = attack.attack_batch(inputs, targets)\n if type(const) is list: \n const = const[0]\n if len(adv.shape) == 3:\n adv = adv.reshape((1,) + adv.shape)\n timeend = time.time()\n l2_distortion = np.sum((adv-inputs)**2)**.5\n adversarial_predict = model.model.predict(adv)\n adversarial_predict = np.squeeze(adversarial_predict)\n adversarial_prob = np.sort(adversarial_predict)\n adversarial_class = np.argsort(adversarial_predict)\n print(\"adversarial probabilities:\", adversarial_prob[-1:-6:-1])\n print(\"adversarial classification:\", adversarial_class[-1:-6:-1])\n success = False\n if args['untargeted']:\n if adversarial_class[-1] != original_class[-1]:\n success = True\n else:\n if adversarial_class[-1] == np.argmax(targets):\n success = True\n if l2_distortion > 20.0:\n success = False\n if success:\n total_success += 1\n l2_total += l2_distortion\n suffix = \"id{}_seq{}_prev{}_adv{}_{}_dist{}\".format(all_true_ids[i], i, original_class[-1], adversarial_class[-1], success, l2_distortion)\n print(\"Saving to\", suffix)\n show(inputs, \"{}/{}/{}_original_{}.png\".format(args['save'], args['dataset'], img_no, suffix))\n show(adv, \"{}/{}/{}_adversarial_{}.png\".format(args['save'], args['dataset'], img_no, suffix))\n show(adv - inputs, \"{}/{}/{}_diff_{}.png\".format(args['save'], args['dataset'], img_no, suffix))\n print(\"[STATS][L1] total = {}, seq = {}, id = {}, time = {:.3f}, success = {}, const = {:.6f}, prev_class = {}, new_class = {}, distortion = {:.5f}, success_rate = {:.3f}, l2_avg = {:.5f}\".format(img_no, i, all_true_ids[i], timeend - timestart, success, const, original_class[-1], adversarial_class[-1], l2_distortion, total_success / float(img_no), 0 if total_success == 0 else l2_total / total_success))\n sys.stdout.flush()\n\n # t = np.random.randn(28*28).reshape(1,28,28,1)\n # print(model.model.predict(t))\n\nif __name__ == \"__main__\":\n import argparse\n parser = argparse.ArgumentParser()\n parser.add_argument(\"-d\", \"--dataset\", choices=[\"mnist\", \"cifar10\", \"imagenet\"], default=\"mnist\")\n parser.add_argument(\"-s\", \"--save\", default=\"./saved_results\")\n parser.add_argument(\"-a\", \"--attack\", choices=[\"white\", \"black\"], default=\"white\")\n parser.add_argument(\"-n\", \"--numimg\", type=int, default=0, help = \"number of test images to attack\")\n parser.add_argument(\"-m\", \"--maxiter\", type=int, default=0, help = \"set 0 to use default value\")\n parser.add_argument(\"-p\", \"--print_every\", type=int, default=100, help = \"print objs every PRINT_EVERY iterations\")\n parser.add_argument(\"-o\", \"--early_stop_iters\", type=int, default=100, help = \"print objs every EARLY_STOP_ITER iterations, 0 is maxiter//10\")\n parser.add_argument(\"-f\", \"--firstimg\", type=int, default=0)\n parser.add_argument(\"-b\", \"--binary_steps\", type=int, default=0)\n parser.add_argument(\"-c\", \"--init_const\", type=float, default=0.0)\n parser.add_argument(\"-z\", \"--use_zvalue\", action='store_true')\n parser.add_argument(\"-u\", \"--untargeted\", action='store_true')\n parser.add_argument(\"-r\", \"--reset_adam\", action='store_true', help = \"reset adam after an initial solution is found\")\n parser.add_argument(\"--use_resize\", action='store_true', help = \"resize image (only works on imagenet!)\")\n parser.add_argument(\"--adam_beta1\", type=float, default=0.9)\n parser.add_argument(\"--adam_beta2\", type=float, default=0.999)\n parser.add_argument(\"--seed\", type=int, default=1216)\n parser.add_argument(\"--solver\", choices=[\"adam\", \"newton\", \"adam_newton\", \"fake_zero\"], default=\"adam\")\n parser.add_argument(\"--save_ckpts\", default=\"\", help = \"path to save checkpoint file\")\n parser.add_argument(\"--load_ckpt\", default=\"\", help = \"path to numpy checkpoint file\")\n parser.add_argument(\"--start_iter\", default=0, type=int, help = \"iteration number for start, useful when loading a checkpoint\")\n parser.add_argument(\"--init_size\", default=32, type=int, help = \"starting with this size when --use_resize\")\n parser.add_argument(\"--uniform\", action='store_true', help = \"disable importance sampling\")\n args = vars(parser.parse_args())\n # add some additional parameters\n # learning rate\n args['lr'] = 1e-2\n args['inception'] = False\n args['use_tanh'] = True\n # args['use_resize'] = False\n if args['maxiter'] == 0:\n if args['attack'] == \"white\":\n args['maxiter'] = 1000\n else:\n if args['dataset'] == \"imagenet\":\n if args['untargeted']:\n args['maxiter'] = 1500\n else:\n args['maxiter'] = 50000\n elif args['dataset'] == \"mnist\":\n args['maxiter'] = 3000\n else:\n args['maxiter'] = 1000\n if args['init_const'] == 0.0:\n if args['binary_steps'] != 0:\n args['init_const'] = 0.01\n else:\n args['init_const'] = 0.5\n if args['binary_steps'] == 0:\n args['binary_steps'] = 1\n # set up some parameters based on datasets\n if args['dataset'] == \"imagenet\":\n args['inception'] = True\n args['lr'] = 2e-3\n # args['use_resize'] = True\n # args['save_ckpts'] = True\n # for mnist, using tanh causes gradient to vanish\n if args['dataset'] == \"mnist\":\n args['use_tanh'] = False\n # when init_const is not specified, use a reasonable default\n if args['init_const'] == 0.0:\n if args['binary_search']:\n args['init_const'] = 0.01\n else:\n args['init_const'] = 0.5\n # setup random seed\n random.seed(args['seed'])\n np.random.seed(args['seed'])\n print(args)\n main(args)\n\n" }, { "path": "test_attack.py", "content": "## test_attack.py -- sample code to test attack procedure\n##\n## Copyright (C) IBM Corp, 2017-2018\n## Copyright (C) 2016, Nicholas Carlini .\n##\n## This program is licenced under the BSD 2-Clause licence,\n## contained in the LICENCE file in this directory.\n\nimport tensorflow as tf\nimport numpy as np\nimport random\nimport time\n\nfrom setup_cifar import CIFAR, CIFARModel\nfrom setup_mnist import MNIST, MNISTModel\nfrom setup_inception import ImageNet, InceptionModel\n\nfrom l2_attack import CarliniL2\nfrom l0_attack import CarliniL0\nfrom li_attack import CarliniLi\n\n\nfrom PIL import Image\n\n\ndef show(img, name = \"output.png\"):\n \"\"\"\n Show MNSIT digits in the console.\n \"\"\"\n np.save('img', img)\n fig = (img + 0.5)*255\n fig = fig.astype(np.uint8).squeeze()\n pic = Image.fromarray(fig)\n # pic.resize((512,512), resample=PIL.Image.BICUBIC)\n pic.save(name)\n remap = \" .*#\"+\"#\"*100\n img = (img.flatten()+.5)*3\n if len(img) != 784: return\n print(\"START\")\n for i in range(28):\n print(\"\".join([remap[int(round(x))] for x in img[i*28:i*28+28]]))\n\ndef generate_data(data, samples, targeted=True, start=0, inception=False):\n \"\"\"\n Generate the input data to the attack algorithm.\n\n data: the images to attack\n samples: number of samples to use\n targeted: if true, construct targeted attacks, otherwise untargeted attacks\n start: offset into data to use\n inception: if targeted and inception, randomly sample 100 targets intead of 1000\n \"\"\"\n inputs = []\n targets = []\n for i in range(samples):\n if targeted:\n if inception:\n seq = random.sample(range(1,1001), 10)\n else:\n seq = range(data.test_labels.shape[1])\n\n print ('image label:', np.argmax(data.test_labels[start+i]))\n for j in seq:\n # skip the original image label\n if (j == np.argmax(data.test_labels[start+i])) and (inception == False):\n continue\n inputs.append(data.test_data[start+i])\n targets.append(np.eye(data.test_labels.shape[1])[j])\n else:\n inputs.append(data.test_data[start+i])\n targets.append(data.test_labels[start+i])\n\n inputs = np.array(inputs)\n targets = np.array(targets)\n\n return inputs, targets\n\n\nif __name__ == \"__main__\":\n with tf.Session() as sess:\n use_log = False\n print('Loading model...')\n # data, model = MNIST(), MNISTModel(\"models/mnist\", sess, use_log)\n # data, model = MNIST(), MNISTModel(\"models/mnist-distilled-100\", sess, use_log)\n # data, model = CIFAR(), CIFARModel(\"models/cifar\", sess, use_log)\n data, model = ImageNet(), InceptionModel(sess, use_log)\n print('Done...')\n batch_size = 1\n if isinstance(model, InceptionModel):\n batch_size = 10\n attack = CarliniL2(sess, model, batch_size=batch_size, initial_const = 1.0, max_iterations=1000, confidence=0, use_log=use_log)\n\n print('Generate data')\n inputs, targets = generate_data(data, samples=1, targeted=True,\n start=6, inception=isinstance(model, InceptionModel))\n print('Done...')\n print(inputs.shape)\n inputs = inputs[0:batch_size]\n targets = targets[0:batch_size]\n timestart = time.time()\n adv = attack.attack(inputs, targets)\n timeend = time.time()\n \n print(\"Took\",timeend-timestart,\"seconds to run\",len(inputs),\"samples.\")\n\n for i in range(len(adv)):\n print(\"Valid:\")\n show(inputs[i], \"original_{}.png\".format(i))\n print(\"Classification:\", np.argsort(model.model.predict(inputs[i:i+1]))[-1:-6:-1])\n print(\"Target:\", np.argmax(targets[i]))\n print(\"Adversarial:\")\n show(adv[i], \"adversarial_{}.png\".format(i))\n show(adv[i] - inputs[i], \"attack_diff.png\")\n \n print(\"Classification:\", np.argsort(model.model.predict(adv[i:i+1]))[-1:-6:-1])\n\n print(\"Total distortion:\", np.sum((adv[i]-inputs[i])**2)**.5)\n\n # t = np.random.randn(28*28).reshape(1,28,28,1)\n # print(model.model.predict(t))\n\n" }, { "path": "test_attack_black.py", "content": "## test_attack.py -- sample code to test attack procedure\n##\n## Copyright (C) IBM Corp, 2017-2018\n## Copyright (C) 2016, Nicholas Carlini .\n##\n## This program is licenced under the BSD 2-Clause licence,\n## contained in the LICENCE file in this directory.\n\nimport tensorflow as tf\nimport numpy as np\nimport time\n\nfrom setup_cifar import CIFAR, CIFARModel\nfrom setup_mnist import MNIST, MNISTModel\nfrom setup_inception import ImageNet, InceptionModel\n\nfrom l2_attack import CarliniL2\nfrom l0_attack import CarliniL0\nfrom li_attack import CarliniLi\nfrom l2_attack_black import BlackBoxL2\n\nfrom PIL import Image\n\n\ndef show(img, name = \"output.png\"):\n \"\"\"\n Show MNSIT digits in the console.\n \"\"\"\n np.save('img', img)\n fig = (img + 0.5)*255\n fig = fig.astype(np.uint8).squeeze()\n pic = Image.fromarray(fig)\n # pic.resize((512,512), resample=PIL.Image.BICUBIC)\n pic.save(name)\n remap = \" .*#\"+\"#\"*100\n img = (img.flatten()+.5)*3\n if len(img) != 784: return\n print(\"START\")\n for i in range(28):\n print(\"\".join([remap[int(round(x))] for x in img[i*28:i*28+28]]))\n\n\ndef generate_data(data, samples, targeted=True, start=0, inception=False):\n \"\"\"\n Generate the input data to the attack algorithm.\n\n data: the images to attack\n samples: number of samples to use\n targeted: if true, construct targeted attacks, otherwise untargeted attacks\n start: offset into data to use\n inception: if targeted and inception, randomly sample 100 targets intead of 1000\n \"\"\"\n inputs = []\n targets = []\n for i in range(samples):\n if targeted:\n if inception:\n seq = random.sample(range(1,1001), 10)\n else:\n seq = range(data.test_labels.shape[1])\n\n # print ('image label:', np.argmax(data.test_labels[start+i]))\n for j in seq:\n # skip the original image label\n if (j == np.argmax(data.test_labels[start+i])) and (inception == False):\n continue\n inputs.append(data.test_data[start+i])\n targets.append(np.eye(data.test_labels.shape[1])[j])\n else:\n inputs.append(data.test_data[start+i])\n targets.append(data.test_labels[start+i])\n\n inputs = np.array(inputs)\n targets = np.array(targets)\n\n return inputs, targets\n\n\nif __name__ == \"__main__\":\n with tf.Session() as sess:\n use_log = True\n # data, model = MNIST(), MNISTModel(\"models/mnist\", sess, use_log)\n # data, model = CIFAR(), CIFARModel(\"models/cifar\", sess, use_log)\n data, model = ImageNet(), InceptionModel(sess, use_log)\n attack = BlackBoxL2(sess, model, batch_size=128, max_iterations=15000, confidence=0, use_log=use_log)\n\n inputs, targets = generate_data(data, samples=1, targeted=True,\n start=6, inception=False)\n inputs = inputs[1:2]\n targets = targets[1:2]\n timestart = time.time()\n adv = attack.attack(inputs, targets)\n timeend = time.time()\n \n print(\"Took\",timeend-timestart,\"seconds to run\",len(inputs),\"samples.\")\n\n print(\"Valid:\")\n show(inputs[0], \"original.png\")\n print(\"Adversarial:\")\n show(adv, \"adversarial.png\")\n show(adv - inputs[0], \"diff.png\")\n \n print(\"Valid Classification:\", np.argsort(model.model.predict(inputs[0].reshape((1,) + adv.shape)))[-1:-6:-1])\n print(\"Adversarial Classification:\", np.argsort(model.model.predict(adv.reshape((1,) + adv.shape)))[-1:-6:-1])\n\n print(\"Total distortion:\", np.sum((adv-inputs[0])**2)**.5)\n" }, { "path": "train_models.py", "content": "## train_models.py -- train the neural network models for attacking\n##\n## Copyright (C) IBM Corp, 2017-2018\n## Copyright (C) 2016, Nicholas Carlini .\n##\n## This program is licenced under the BSD 2-Clause licence,\n## contained in the LICENCE file in this directory.\n\n\nimport numpy as np\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Activation, Flatten\nfrom keras.layers import Conv2D, MaxPooling2D\nfrom keras.optimizers import SGD\n\nimport tensorflow as tf\nfrom setup_mnist import MNIST\nfrom setup_cifar import CIFAR\nimport os\n\ndef train(data, file_name, params, num_epochs=50, batch_size=128, train_temp=1, init=None):\n \"\"\"\n Standard neural network training procedure.\n \"\"\"\n model = Sequential()\n\n print(data.train_data.shape)\n \n model.add(Conv2D(params[0], (3, 3),\n input_shape=data.train_data.shape[1:]))\n model.add(Activation('relu'))\n model.add(Conv2D(params[1], (3, 3)))\n model.add(Activation('relu'))\n model.add(MaxPooling2D(pool_size=(2, 2)))\n\n model.add(Conv2D(params[2], (3, 3)))\n model.add(Activation('relu'))\n model.add(Conv2D(params[3], (3, 3)))\n model.add(Activation('relu'))\n model.add(MaxPooling2D(pool_size=(2, 2)))\n\n model.add(Flatten())\n model.add(Dense(params[4]))\n model.add(Activation('relu'))\n model.add(Dropout(0.5))\n model.add(Dense(params[5]))\n model.add(Activation('relu'))\n model.add(Dense(10))\n \n if init != None:\n model.load_weights(init)\n\n def fn(correct, predicted):\n return tf.nn.softmax_cross_entropy_with_logits(labels=correct,\n logits=predicted/train_temp)\n\n sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)\n \n model.compile(loss=fn,\n optimizer=sgd,\n metrics=['accuracy'])\n \n model.fit(data.train_data, data.train_labels,\n batch_size=batch_size,\n validation_data=(data.validation_data, data.validation_labels),\n nb_epoch=num_epochs,\n shuffle=True)\n \n\n if file_name != None:\n model.save(file_name)\n\n return model\n\ndef train_distillation(data, file_name, params, num_epochs=50, batch_size=128, train_temp=1):\n \"\"\"\n Train a network using defensive distillation.\n\n Distillation as a Defense to Adversarial Perturbations against Deep Neural Networks\n Nicolas Papernot, Patrick McDaniel, Xi Wu, Somesh Jha, Ananthram Swami\n IEEE S&P, 2016.\n \"\"\"\n if not os.path.exists(file_name+\"_init\"):\n # Train for one epoch to get a good starting point.\n train(data, file_name+\"_init\", params, 1, batch_size)\n \n # now train the teacher at the given temperature\n teacher = train(data, file_name+\"_teacher\", params, num_epochs, batch_size, train_temp,\n init=file_name+\"_init\")\n\n # evaluate the labels at temperature t\n predicted = teacher.predict(data.train_data)\n with tf.Session() as sess:\n y = sess.run(tf.nn.softmax(predicted/train_temp))\n print(y)\n data.train_labels = y\n\n # train the student model at temperature t\n student = train(data, file_name, params, num_epochs, batch_size, train_temp,\n init=file_name+\"_init\")\n\n # and finally we predict at temperature 1\n predicted = student.predict(data.train_data)\n\n print(predicted)\n \nif not os.path.isdir('models'):\n os.makedirs('models')\n\ntrain(CIFAR(), \"models/cifar\", [64, 64, 128, 128, 256, 256], num_epochs=50)\ntrain(MNIST(), \"models/mnist\", [32, 32, 64, 64, 200, 200], num_epochs=50)\n\ntrain_distillation(MNIST(), \"models/mnist-distilled-100\", [32, 32, 64, 64, 200, 200],\n num_epochs=50, train_temp=100)\ntrain_distillation(CIFAR(), \"models/cifar-distilled-100\", [64, 64, 128, 128, 256, 256],\n num_epochs=50, train_temp=100)\n" }, { "path": "verify.py", "content": "## verify.py -- check the accuracy of a neural network\n##\n## Copyright (C) IBM Corp, 2017-2018\n## Copyright (C) 2016, Nicholas Carlini .\n##\n## This program is licenced under the BSD 2-Clause licence,\n## contained in the LICENCE file in this directory.\n\nfrom setup_cifar import CIFAR, CIFARModel\nfrom setup_mnist import MNIST, MNISTModel\nfrom setup_inception import ImageNet, InceptionModel\n\nimport tensorflow as tf\nimport numpy as np\n\nBATCH_SIZE = 1\n\nwith tf.Session() as sess:\n data, model = MNIST(), MNISTModel(\"models/mnist\", sess)\n data, model = CIFAR(), CIFARModel(\"models/cifar\", sess)\n data, model = ImageNet(), InceptionModel(sess)\n\n x = tf.placeholder(tf.float32, (None, model.image_size, model.image_size, model.num_channels))\n y = model.predict(x)\n\n r = []\n for i in range(0,len(data.test_data),BATCH_SIZE):\n pred = sess.run(y, {x: data.test_data[i:i+BATCH_SIZE]})\n #print(pred)\n #print('real',data.test_labels[i],'pred',np.argmax(pred))\n r.append(np.argmax(pred,1) == np.argmax(data.test_labels[i:i+BATCH_SIZE],1))\n print(np.mean(r))\n" } ]