Repository: labsix/limited-blackbox-attacks
Branch: master
Commit: 076eebdbd970
Files: 13
Total size: 55.9 KB

Directory structure:
gitextract_jco60r8s/

├── .gitignore
├── README.md
├── attacks.py
├── label-only.sh
├── main.py
├── partial-info.sh
├── precompute.py
├── query-limited.sh
└── tools/
    ├── __init__.py
    ├── imagenet_labels.py
    ├── inception_v3_imagenet.py
    ├── logging_utils.py
    └── utils.py

================================================
FILE CONTENTS
================================================

================================================
FILE: .gitignore
================================================
**/data/
partial_info_*/
run_gpu_*
**/*.pyc
partial_info_*/
ptl_info_*


================================================
FILE: README.md
================================================
# Code for Black-box Adversarial Attacks with Limited Queries and Information
Codebase for reproducing the results in the paper "Black-Box Adversarial Attacks with Limited Queries and Information". The paper can be found [on arxiv](http://arxiv.org/abs/1804.08598), and our explanatory blog post can be found on [labsix.org](http://labsix.org).

To reproduce our results:

1. Make a directory `tools/data`, and in it put the decompressed Inceptionv3 classifier from (http://download.tensorflow.org/models/inception_v3_2016_08_28.tar.gz)
2. Set `IMAGENET_PATH` in main.py, attacks.py, and precompute.py to the location of the ImageNet dataset on your machine.
3. Precompute the starting images (for partial-information and label-only attacks) with `python precompute.py`
4. Run the reproduction scripts with `{query-limited|partial-info|label-only}.sh`, making sure to first edit them specifying an img-index (by default runs on imagenet image 0) 

## Citation
```
@inproceedings{ilyas2018blackbox,
  author = {Andrew Ilyas and Logan Engstrom and Anish Athalye and Jessy Lin},
  title = {Black-box Adversarial Attacks with Limited Queries and Information},
  booktitle = {Proceedings of the 35th International Conference on Machine Learning, {ICML} 2018},
  year = {2018},
  month = jul,
  url = {https://arxiv.org/abs/1804.08598},
}
```


================================================
FILE: attacks.py
================================================
from PIL import Image
import numpy as np
import tensorflow as tf
from tools.utils import *
import json
import pdb
import os
import sys
import time
import scipy.misc
import PIL

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from tools.logging_utils import *

from tools.inception_v3_imagenet import model
from tools.imagenet_labels import label_to_name

IMAGENET_PATH=""
NUM_LABELS=1000
SIZE = 299

def main(args, gpus):
    # INITIAL IMAGE AND CLASS SELECTION
    if args.img_path:
        initial_img = np.asarray(Image.open(args.img_path).resize((SIZE, SIZE)))
        orig_class = args.orig_class
        initial_img = initial_img.astype(np.float32) / 255.0
    else:
        x, y = get_image(args.img_index, IMAGENET_PATH)
        orig_class = y
        initial_img = x

    # PARAMETER SETUP
    if args.target_class is None:
        target_class = pseudorandom_target(args.img_index, NUM_LABELS, orig_class)
        print('chose pseudorandom target class: %d' % target_class)
    else:
        target_class = args.target_class
    batch_size = args.batch_size
    out_dir = args.out_dir
    epsilon = args.epsilon
    lower = np.clip(initial_img - args.epsilon, 0., 1.)
    upper = np.clip(initial_img + args.epsilon, 0., 1.)
    adv = initial_img.copy() if not args.restore else \
            np.clip(np.load(args.restore), lower, upper)
    batch_per_gpu = batch_size // len(gpus)
    log_iters = args.log_iters
    queries_per_iter = args.samples_per_draw
    max_iters = int(np.ceil(args.max_queries // queries_per_iter))
    max_lr = args.max_lr
    # ----- partial info params -----
    k = args.top_k
    goal_epsilon = epsilon
    adv_thresh = args.adv_thresh
    if k > 0:
        if target_class == -1:
            raise ValueError("Partial-information attack is a targeted attack.")
        adv = image_of_class(target_class, IMAGENET_PATH)
        epsilon = args.starting_eps
        delta_epsilon = args.starting_delta_eps
    else:
        k = NUM_LABELS
    # ----- label only params -----
    label_only = args.label_only
    zero_iters = args.zero_iters

    # TARGET CLASS SELECTION
    if target_class < 0:
        one_hot_vec = one_hot(orig_class, NUM_LABELS)
    else:
        one_hot_vec = one_hot(target_class, NUM_LABELS)
    labels = np.repeat(np.expand_dims(one_hot_vec, axis=0),
                       repeats=batch_per_gpu, axis=0)
    is_targeted = 1 if target_class >= 0 else -1

    # SESSION INITIALIZATION
    sess = tf.InteractiveSession()
    x = tf.placeholder(tf.float32, initial_img.shape)
    eval_logits, eval_preds = model(sess, tf.expand_dims(x, 0))
    if target_class >= 0:
        eval_percent_adv = tf.equal(eval_preds[0], tf.constant(target_class, tf.int64))
    else:
        eval_percent_adv = tf.not_equal(eval_preds[0], tf.constant(orig_class, tf.int64))

    # TENSORBOARD SETUP
    empirical_loss = tf.placeholder(dtype=tf.float32, shape=())
    lr_placeholder = tf.placeholder(dtype=tf.float32, shape=())
    loss_vs_queries = tf.summary.scalar('empirical loss vs queries', empirical_loss)
    loss_vs_steps = tf.summary.scalar('empirical loss vs step', empirical_loss)
    lr_vs_queries = tf.summary.scalar('lr vs queries', lr_placeholder)
    lr_vs_steps = tf.summary.scalar('lr vs step', lr_placeholder)
    writer = tf.summary.FileWriter(out_dir, graph=sess.graph)
    log_file = open(os.path.join(out_dir, 'log.txt'), 'w+')
    with open(os.path.join(out_dir, 'args.json'), 'w') as args_file:
        json.dump(args.__dict__, args_file)

    # LOSS FUNCTION
    def standard_loss(eval_points, noise):
        logits, preds = model(sess, eval_points)
        losses = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels)
        return losses, noise

    def label_only_loss(eval_points, noise):
        noised_eval_points = tf.zeros((batch_per_gpu,))
        tiled_points = tf.tile(tf.expand_dims(eval_points, 0), [zero_iters,1,1,1,1])
        noised_eval_im = tiled_points + \
                tf.random_uniform(tf.shape(tiled_points), minval=-1, \
                maxval=1)*args.label_only_sigma
        logits, preds = model(sess, tf.reshape(noised_eval_im, (-1,) + initial_img.shape))
        vals, inds = tf.nn.top_k(logits, k=k)
        real_inds = tf.reshape(inds, (zero_iters, batch_per_gpu, -1))
        rank_range = tf.range(start=k, limit=0, delta=-1, dtype=tf.float32)
        tiled_rank_range = tf.tile(tf.reshape(rank_range, (1, 1, k)), [zero_iters, batch_per_gpu, 1])
        batches_in = tf.where(tf.equal(real_inds, target_class), 
                tiled_rank_range, tf.zeros(tf.shape(tiled_rank_range)))
        return 1 - tf.reduce_mean(batches_in, [0, 2]), noise

    def partial_info_loss(eval_points, noise):
        logits, preds = model(sess, eval_points)
        losses = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels)
        vals, inds = tf.nn.top_k(logits, k=k)
        # inds is batch_size x k
        good_inds = tf.where(tf.equal(inds, tf.constant(target_class))) # returns (# true) x 3
        good_images = good_inds[:,0] # inds of img in batch that worked
        losses = tf.gather(losses, good_images)
        noise = tf.gather(noise, good_images)
        return losses, noise

    # GRADIENT ESTIMATION GRAPH
    grad_estimates = []
    final_losses = []
    loss_fn = label_only_loss if label_only else \
                (partial_info_loss if k < NUM_LABELS else standard_loss)
    for i, device in enumerate(gpus):
        with tf.device(device):
            print('loading on gpu %d of %d' % (i+1, len(gpus)))
            noise_pos = tf.random_normal((batch_per_gpu//2,) + initial_img.shape)
            noise = tf.concat([noise_pos, -noise_pos], axis=0)
            eval_points = x + args.sigma * noise
            losses, noise = loss_fn(eval_points, noise)
        losses_tiled = tf.tile(tf.reshape(losses, (-1, 1, 1, 1)), (1,) + initial_img.shape)
        grad_estimates.append(tf.reduce_mean(losses_tiled * noise, axis=0)/args.sigma)
        final_losses.append(losses)
    grad_estimate = tf.reduce_mean(grad_estimates, axis=0)
    final_losses = tf.concat(final_losses, axis=0)

    # GRADIENT ESTIMATION EVAL
    def get_grad(pt, spd, bs):
        num_batches = spd // bs
        losses = []
        grads = []
        feed_dict = {x: pt}
        for _ in range(num_batches):
            loss, dl_dx_ = sess.run([final_losses, grad_estimate], feed_dict)
            losses.append(np.mean(loss))
            grads.append(dl_dx_)
        return np.array(losses).mean(), np.mean(np.array(grads), axis=0)

    # CONCURRENT VISUALIZATION
    if args.visualize:
        with tf.device('/cpu:0'):
            render_feed = tf.placeholder(tf.float32, initial_img.shape)
            render_exp = tf.expand_dims(render_feed, axis=0)
            render_logits, _ = model(sess, render_exp)

    assert out_dir[-1] == '/'

    # HISTORY VARIABLES (for backtracking and momentum)
    num_queries = 0
    g = 0
    prev_adv = adv
    last_ls = []

    # STEP CONDITION (important for partial-info attacks)
    def robust_in_top_k(t_, prop_adv_,k_):
        if k == NUM_LABELS:
            return True
        for i in range(1):
            n = np.random.rand(*prop_adv_.shape)*args.sigma
            eval_logits_ = sess.run(eval_logits, {x: prop_adv_})[0]
            if not t_ in eval_logits_.argsort()[-k_:][::-1]:
                return False
        return True
        

    # MAIN LOOP
    for i in range(max_iters):
        start = time.time()
        if args.visualize:
            render_frame(sess, adv, i, render_logits, render_feed, out_dir)

        # CHECK IF WE SHOULD STOP
        padv = sess.run(eval_percent_adv, feed_dict={x: adv})
        if padv == 1 and epsilon <= goal_epsilon:
            print('[log] early stopping at iteration %d' % i)
            break

        prev_g = g
        l, g = get_grad(adv, args.samples_per_draw, batch_size)

        # SIMPLE MOMENTUM
        g = args.momentum * prev_g + (1.0 - args.momentum) * g

        # PLATEAU LR ANNEALING
        last_ls.append(l)
        last_ls = last_ls[-args.plateau_length:]
        if last_ls[-1] > last_ls[0] \
           and len(last_ls) == args.plateau_length:
            if max_lr > args.min_lr:
                print("[log] Annealing max_lr")
                max_lr = max(max_lr / args.plateau_drop, args.min_lr)
            last_ls = []

        # SEARCH FOR LR AND EPSILON DECAY
        current_lr = max_lr
        proposed_adv = adv - is_targeted * current_lr * np.sign(g)
        prop_de = 0.0
        if l < adv_thresh and epsilon > goal_epsilon:
            prop_de = delta_epsilon
        while current_lr >= args.min_lr:
            # PARTIAL INFORMATION ONLY
            if k < NUM_LABELS:
                proposed_epsilon = max(epsilon - prop_de, goal_epsilon)
                lower = np.clip(initial_img - proposed_epsilon, 0, 1)
                upper = np.clip(initial_img + proposed_epsilon, 0, 1)
            # GENERAL LINE SEARCH
            proposed_adv = adv - is_targeted * current_lr * np.sign(g)
            proposed_adv = np.clip(proposed_adv, lower, upper)
            num_queries += 1
            if robust_in_top_k(target_class, proposed_adv, k):
                if prop_de > 0:
                    delta_epsilon = max(prop_de, 0.1)
                    last_ls = []
                prev_adv = adv
                adv = proposed_adv
                epsilon = max(epsilon - prop_de/args.conservative, goal_epsilon)
                break
            elif current_lr >= args.min_lr*2:
                current_lr = current_lr / 2
                #print("[log] backtracking lr to %3f" % (current_lr,))
            else:
                prop_de = prop_de / 2
                if prop_de == 0:
                    raise ValueError("Did not converge.")
                if prop_de < 2e-3:
                    prop_de = 0
                current_lr = max_lr
                print("[log] backtracking eps to %3f" % (epsilon-prop_de,))

        # BOOK-KEEPING STUFF
        num_queries += args.samples_per_draw*(zero_iters if label_only else 1)
        log_text = 'Step %05d: loss %.4f lr %.2E eps %.3f (time %.4f)' % (i, l, \
                        current_lr, epsilon, time.time() - start)
        log_file.write(log_text + '\n')
        print(log_text)

        if i % log_iters == 0:
            lvq, lvs, lrvq, lrvs = sess.run([loss_vs_queries, loss_vs_steps,
                                             lr_vs_queries, lr_vs_steps], {
                                                 empirical_loss:l,
                                                 lr_placeholder:current_lr
                                             })
            writer.add_summary(lvq, num_queries)
            writer.add_summary(lrvq, num_queries)
            writer.add_summary(lvs, i)
            writer.add_summary(lrvs, i)

        if (i+1) % args.save_iters == 0 and args.save_iters > 0:
            np.save(os.path.join(out_dir, '%s.npy' % (i+1)), adv)
            scipy.misc.imsave(os.path.join(out_dir, '%s.png' % (i+1)), adv)
    log_output(sess, eval_logits, eval_preds, x, adv, initial_img, \
            target_class, out_dir, orig_class, num_queries)

if __name__ == '__main__':
    main()


================================================
FILE: label-only.sh
================================================
python main.py --max-queries 1000000 --top-k 1 --img-index 0 --out-dir label_only/ --adv-thresh 0.15 --sigma 1e-3 --plateau-length 20 --samples-per-draw 100 --batch-size 4 --min-lr 1e-4 --label-only --zero-iters 50 --label-only-sigma 2e-3 --conservative 4 


================================================
FILE: main.py
================================================
import os
import json
import shutil
import argparse
from tensorflow.python.client import device_lib

import attacks 

BATCH_SIZE = 50
SIGMA = 1e-3
EPSILON = 0.05
SAMPLES_PER_DRAW = 50
LEARNING_RATE = 1e-2
LOG_ITERS_FACTOR = 2
IMAGENET_PATH = ''

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--samples-per-draw', type=int, default=SAMPLES_PER_DRAW)
    parser.add_argument('--batch-size', type=int, default=BATCH_SIZE)
    parser.add_argument('--target-class', type=int, help='negative => untargeted')
    parser.add_argument('--orig-class', type=int)
    parser.add_argument('--sigma', type=float, default=SIGMA)
    parser.add_argument('--epsilon', type=float, default=EPSILON)
    parser.add_argument('--img-path', type=str)
    parser.add_argument('--img-index', type=int)
    parser.add_argument('--out-dir', type=str, required=True,
                        help='dir to save to if not gridding; otherwise parent \
                        dir of grid directories')
    parser.add_argument('--log-iters', type=int, default=1)
    parser.add_argument('--restore', type=str, help='restore path of img')
    parser.add_argument('--momentum', type=float, default=0.9)
    parser.add_argument('--max-queries', type=int, default=10000)
    parser.add_argument('--save-iters', type=int, default=50)
    parser.add_argument('--plateau-drop', type=float, default=2.0)
    parser.add_argument('--min-lr-ratio', type=int, default=200)
    parser.add_argument('--plateau-length', type=int, default=5)
    parser.add_argument('--gpus', type=int, help='number of GPUs to use')
    parser.add_argument('--imagenet-path', type=str)
    parser.add_argument('--visualize', action='store_true')
    parser.add_argument('--max-lr', type=float, default=1e-2)
    parser.add_argument('--min-lr', type=float, default=5e-5)
    # PARTIAL INFORMATION ARGUMENTS
    parser.add_argument('--top-k', type=int, default=-1)
    parser.add_argument('--adv-thresh', type=float, default=-1.0)
    # LABEL ONLY ARGUMENTS
    parser.add_argument('--label-only', action='store_true')
    parser.add_argument('--zero-iters', type=int, default=100, help="how many points to use for the proxy score")
    parser.add_argument('--label-only-sigma', type=float, default=1e-3, help="distribution width for proxy score")
    parser.add_argument('--starting-eps', type=float, default=1.0)
    parser.add_argument('--starting-delta-eps', type=float, default=0.5)
    parser.add_argument('--min-delta-eps', type=float, default=0.1)
    parser.add_argument('--conservative', type=int, default=2, help="How conservative we should be in epsilon decay; increase if no convergence")
    args = parser.parse_args()

    # Data checks
    if not (args.img_path is None and args.img_index is not None or
            args.img_path is not None and args.img_index is None):
        raise ValueError('can only set one of img-path, img-index')
    if args.img_path and not (args.orig_class or args.target_class):
        raise ValueError('orig and target class required with image path')
    if (args.target_class is None and args.img_index is None):
        raise ValueError('must give target class if not using index')
    assert args.samples_per_draw % args.batch_size == 0
    gpus = get_available_gpus()
    if args.gpus:
        if args.gpus > len(gpus):
            raise RuntimeError('not enough GPUs! (requested %d, found %d)' % (args.gpus, len(gpus)))
        gpus = gpus[:args.gpus]
    if not gpus:
        raise NotImplementedError('no support for using CPU-only because lazy')
    if args.batch_size % 2*len(gpus) != 0:
        raise ValueError('batch size must be divisible by 2 * number of GPUs (batch_size=%d, gpus=%d)' % (
            batch_size,
            len(gpus)
        ))

    # CLEAR THE PATH
    if os.path.exists(args.out_dir):
        shutil.rmtree(args.out_dir)
    os.makedirs(args.out_dir)

    # PRINT PARAMS
    args_text = json.dumps(args.__dict__)
    print(args_text)
    attacks.main(args, gpus)

def get_available_gpus():
    local_device_protos = device_lib.list_local_devices()
    return [x.name for x in local_device_protos if x.device_type == 'GPU']

if __name__ == "__main__":
    main()


================================================
FILE: partial-info.sh
================================================
python main.py --max-queries 1000000 --top-k 1 --img-index 0 --out-dir label_only/ --adv-thresh 0.2 --sigma 1e-5 --plateau-length 20 --samples-per-draw 100 --batch-size 100 --min-lr 1e-3


================================================
FILE: precompute.py
================================================
from tools import utils
from tools.inception_v3_imagenet import model
import tensorflow as tf
import pickle
import sys

IMAGENET_PATH=""
if __name__=="__main__":
    if IMAGENET_PATH == "":
        raise ValueError("Please open precompute.py and set IMAGENET_PATH")
    s = (299, 299, 3)
    dataset = sys.argv[1]
    last_j = 0
    sess = tf.InteractiveSession()
    x = tf.placeholder(tf.float32, s)
    _, preds = model(sess, tf.expand_dims(x, 0))
    label_dict = {}
    for i in range(1,1000):
        print("Looking for %d" % (i,))
        if i in label_dict:
            continue
        for j in range(last_j, 50000):
            im, lab = utils.get_image(j, IMAGENET_PATH)
            if sess.run(preds, {x: im})[0] == lab:
                label_dict[lab] = j
            if lab == i:
                label_dict[i] = j
                break
        last_j = j
    pickle.dump(label_dict, open("tools/data/imagenet.pickle", "wb"))


================================================
FILE: query-limited.sh
================================================
python main.py --max-queries 1000000 --img-index 0 --out-dir query_limited/ 


================================================
FILE: tools/__init__.py
================================================


================================================
FILE: tools/imagenet_labels.py
================================================
_lut = [
    'tench, Tinca tinca',
    'goldfish, Carassius auratus',
    'great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias',
    'tiger shark, Galeocerdo cuvieri',
    'hammerhead, hammerhead shark',
    'electric ray, crampfish, numbfish, torpedo',
    'stingray',
    'cock',
    'hen',
    'ostrich, Struthio camelus',
    'brambling, Fringilla montifringilla',
    'goldfinch, Carduelis carduelis',
    'house finch, linnet, Carpodacus mexicanus',
    'junco, snowbird',
    'indigo bunting, indigo finch, indigo bird, Passerina cyanea',
    'robin, American robin, Turdus migratorius',
    'bulbul',
    'jay',
    'magpie',
    'chickadee',
    'water ouzel, dipper',
    'kite',
    'bald eagle, American eagle, Haliaeetus leucocephalus',
    'vulture',
    'great grey owl, great gray owl, Strix nebulosa',
    'European fire salamander, Salamandra salamandra',
    'common newt, Triturus vulgaris',
    'eft',
    'spotted salamander, Ambystoma maculatum',
    'axolotl, mud puppy, Ambystoma mexicanum',
    'bullfrog, Rana catesbeiana',
    'tree frog, tree-frog',
    'tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui',
    'loggerhead, loggerhead turtle, Caretta caretta',
    'leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea',
    'mud turtle',
    'terrapin',
    'box turtle, box tortoise',
    'banded gecko',
    'common iguana, iguana, Iguana iguana',
    'American chameleon, anole, Anolis carolinensis',
    'whiptail, whiptail lizard',
    'agama',
    'frilled lizard, Chlamydosaurus kingi',
    'alligator lizard',
    'Gila monster, Heloderma suspectum',
    'green lizard, Lacerta viridis',
    'African chameleon, Chamaeleo chamaeleon',
    'Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis',
    'African crocodile, Nile crocodile, Crocodylus niloticus',
    'American alligator, Alligator mississipiensis',
    'triceratops',
    'thunder snake, worm snake, Carphophis amoenus',
    'ringneck snake, ring-necked snake, ring snake',
    'hognose snake, puff adder, sand viper',
    'green snake, grass snake',
    'king snake, kingsnake',
    'garter snake, grass snake',
    'water snake',
    'vine snake',
    'night snake, Hypsiglena torquata',
    'boa constrictor, Constrictor constrictor',
    'rock python, rock snake, Python sebae',
    'Indian cobra, Naja naja',
    'green mamba',
    'sea snake',
    'horned viper, cerastes, sand viper, horned asp, Cerastes cornutus',
    'diamondback, diamondback rattlesnake, Crotalus adamanteus',
    'sidewinder, horned rattlesnake, Crotalus cerastes',
    'trilobite',
    'harvestman, daddy longlegs, Phalangium opilio',
    'scorpion',
    'black and gold garden spider, Argiope aurantia',
    'barn spider, Araneus cavaticus',
    'garden spider, Aranea diademata',
    'black widow, Latrodectus mactans',
    'tarantula',
    'wolf spider, hunting spider',
    'tick',
    'centipede',
    'black grouse',
    'ptarmigan',
    'ruffed grouse, partridge, Bonasa umbellus',
    'prairie chicken, prairie grouse, prairie fowl',
    'peacock',
    'quail',
    'partridge',
    'African grey, African gray, Psittacus erithacus',
    'macaw',
    'sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita',
    'lorikeet',
    'coucal',
    'bee eater',
    'hornbill',
    'hummingbird',
    'jacamar',
    'toucan',
    'drake',
    'red-breasted merganser, Mergus serrator',
    'goose',
    'black swan, Cygnus atratus',
    'tusker',
    'echidna, spiny anteater, anteater',
    'platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus',
    'wallaby, brush kangaroo',
    'koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus',
    'wombat',
    'jellyfish',
    'sea anemone, anemone',
    'brain coral',
    'flatworm, platyhelminth',
    'nematode, nematode worm, roundworm',
    'conch',
    'snail',
    'slug',
    'sea slug, nudibranch',
    'chiton, coat-of-mail shell, sea cradle, polyplacophore',
    'chambered nautilus, pearly nautilus, nautilus',
    'Dungeness crab, Cancer magister',
    'rock crab, Cancer irroratus',
    'fiddler crab',
    'king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica',
    'American lobster, Northern lobster, Maine lobster, Homarus americanus',
    'spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish',
    'crayfish, crawfish, crawdad, crawdaddy',
    'hermit crab',
    'isopod',
    'white stork, Ciconia ciconia',
    'black stork, Ciconia nigra',
    'spoonbill',
    'flamingo',
    'little blue heron, Egretta caerulea',
    'American egret, great white heron, Egretta albus',
    'bittern',
    'crane',
    'limpkin, Aramus pictus',
    'European gallinule, Porphyrio porphyrio',
    'American coot, marsh hen, mud hen, water hen, Fulica americana',
    'bustard',
    'ruddy turnstone, Arenaria interpres',
    'red-backed sandpiper, dunlin, Erolia alpina',
    'redshank, Tringa totanus',
    'dowitcher',
    'oystercatcher, oyster catcher',
    'pelican',
    'king penguin, Aptenodytes patagonica',
    'albatross, mollymawk',
    'grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus',
    'killer whale, killer, orca, grampus, sea wolf, Orcinus orca',
    'dugong, Dugong dugon',
    'sea lion',
    'Chihuahua',
    'Japanese spaniel',
    'Maltese dog, Maltese terrier, Maltese',
    'Pekinese, Pekingese, Peke',
    'Shih-Tzu',
    'Blenheim spaniel',
    'papillon',
    'toy terrier',
    'Rhodesian ridgeback',
    'Afghan hound, Afghan',
    'basset, basset hound',
    'beagle',
    'bloodhound, sleuthhound',
    'bluetick',
    'black-and-tan coonhound',
    'Walker hound, Walker foxhound',
    'English foxhound',
    'redbone',
    'borzoi, Russian wolfhound',
    'Irish wolfhound',
    'Italian greyhound',
    'whippet',
    'Ibizan hound, Ibizan Podenco',
    'Norwegian elkhound, elkhound',
    'otterhound, otter hound',
    'Saluki, gazelle hound',
    'Scottish deerhound, deerhound',
    'Weimaraner',
    'Staffordshire bullterrier, Staffordshire bull terrier',
    'American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier',
    'Bedlington terrier',
    'Border terrier',
    'Kerry blue terrier',
    'Irish terrier',
    'Norfolk terrier',
    'Norwich terrier',
    'Yorkshire terrier',
    'wire-haired fox terrier',
    'Lakeland terrier',
    'Sealyham terrier, Sealyham',
    'Airedale, Airedale terrier',
    'cairn, cairn terrier',
    'Australian terrier',
    'Dandie Dinmont, Dandie Dinmont terrier',
    'Boston bull, Boston terrier',
    'miniature schnauzer',
    'giant schnauzer',
    'standard schnauzer',
    'Scotch terrier, Scottish terrier, Scottie',
    'Tibetan terrier, chrysanthemum dog',
    'silky terrier, Sydney silky',
    'soft-coated wheaten terrier',
    'West Highland white terrier',
    'Lhasa, Lhasa apso',
    'flat-coated retriever',
    'curly-coated retriever',
    'golden retriever',
    'Labrador retriever',
    'Chesapeake Bay retriever',
    'German short-haired pointer',
    'vizsla, Hungarian pointer',
    'English setter',
    'Irish setter, red setter',
    'Gordon setter',
    'Brittany spaniel',
    'clumber, clumber spaniel',
    'English springer, English springer spaniel',
    'Welsh springer spaniel',
    'cocker spaniel, English cocker spaniel, cocker',
    'Sussex spaniel',
    'Irish water spaniel',
    'kuvasz',
    'schipperke',
    'groenendael',
    'malinois',
    'briard',
    'kelpie',
    'komondor',
    'Old English sheepdog, bobtail',
    'Shetland sheepdog, Shetland sheep dog, Shetland',
    'collie',
    'Border collie',
    'Bouvier des Flandres, Bouviers des Flandres',
    'Rottweiler',
    'German shepherd, German shepherd dog, German police dog, alsatian',
    'Doberman, Doberman pinscher',
    'miniature pinscher',
    'Greater Swiss Mountain dog',
    'Bernese mountain dog',
    'Appenzeller',
    'EntleBucher',
    'boxer',
    'bull mastiff',
    'Tibetan mastiff',
    'French bulldog',
    'Great Dane',
    'Saint Bernard, St Bernard',
    'Eskimo dog, husky',
    'malamute, malemute, Alaskan malamute',
    'Siberian husky',
    'dalmatian, coach dog, carriage dog',
    'affenpinscher, monkey pinscher, monkey dog',
    'basenji',
    'pug, pug-dog',
    'Leonberg',
    'Newfoundland, Newfoundland dog',
    'Great Pyrenees',
    'Samoyed, Samoyede',
    'Pomeranian',
    'chow, chow chow',
    'keeshond',
    'Brabancon griffon',
    'Pembroke, Pembroke Welsh corgi',
    'Cardigan, Cardigan Welsh corgi',
    'toy poodle',
    'miniature poodle',
    'standard poodle',
    'Mexican hairless',
    'timber wolf, grey wolf, gray wolf, Canis lupus',
    'white wolf, Arctic wolf, Canis lupus tundrarum',
    'red wolf, maned wolf, Canis rufus, Canis niger',
    'coyote, prairie wolf, brush wolf, Canis latrans',
    'dingo, warrigal, warragal, Canis dingo',
    'dhole, Cuon alpinus',
    'African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus',
    'hyena, hyaena',
    'red fox, Vulpes vulpes',
    'kit fox, Vulpes macrotis',
    'Arctic fox, white fox, Alopex lagopus',
    'grey fox, gray fox, Urocyon cinereoargenteus',
    'tabby, tabby cat',
    'tiger cat',
    'Persian cat',
    'Siamese cat, Siamese',
    'Egyptian cat',
    'cougar, puma, catamount, mountain lion, painter, panther, Felis concolor',
    'lynx, catamount',
    'leopard, Panthera pardus',
    'snow leopard, ounce, Panthera uncia',
    'jaguar, panther, Panthera onca, Felis onca',
    'lion, king of beasts, Panthera leo',
    'tiger, Panthera tigris',
    'cheetah, chetah, Acinonyx jubatus',
    'brown bear, bruin, Ursus arctos',
    'American black bear, black bear, Ursus americanus, Euarctos americanus',
    'ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus',
    'sloth bear, Melursus ursinus, Ursus ursinus',
    'mongoose',
    'meerkat, mierkat',
    'tiger beetle',
    'ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle',
    'ground beetle, carabid beetle',
    'long-horned beetle, longicorn, longicorn beetle',
    'leaf beetle, chrysomelid',
    'dung beetle',
    'rhinoceros beetle',
    'weevil',
    'fly',
    'bee',
    'ant, emmet, pismire',
    'grasshopper, hopper',
    'cricket',
    'walking stick, walkingstick, stick insect',
    'cockroach, roach',
    'mantis, mantid',
    'cicada, cicala',
    'leafhopper',
    'lacewing, lacewing fly',
    "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk",
    'damselfly',
    'admiral',
    'ringlet, ringlet butterfly',
    'monarch, monarch butterfly, milkweed butterfly, Danaus plexippus',
    'cabbage butterfly',
    'sulphur butterfly, sulfur butterfly',
    'lycaenid, lycaenid butterfly',
    'starfish, sea star',
    'sea urchin',
    'sea cucumber, holothurian',
    'wood rabbit, cottontail, cottontail rabbit',
    'hare',
    'Angora, Angora rabbit',
    'hamster',
    'porcupine, hedgehog',
    'fox squirrel, eastern fox squirrel, Sciurus niger',
    'marmot',
    'beaver',
    'guinea pig, Cavia cobaya',
    'sorrel',
    'zebra',
    'hog, pig, grunter, squealer, Sus scrofa',
    'wild boar, boar, Sus scrofa',
    'warthog',
    'hippopotamus, hippo, river horse, Hippopotamus amphibius',
    'ox',
    'water buffalo, water ox, Asiatic buffalo, Bubalus bubalis',
    'bison',
    'ram, tup',
    'bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis',
    'ibex, Capra ibex',
    'hartebeest',
    'impala, Aepyceros melampus',
    'gazelle',
    'Arabian camel, dromedary, Camelus dromedarius',
    'llama',
    'weasel',
    'mink',
    'polecat, fitch, foulmart, foumart, Mustela putorius',
    'black-footed ferret, ferret, Mustela nigripes',
    'otter',
    'skunk, polecat, wood pussy',
    'badger',
    'armadillo',
    'three-toed sloth, ai, Bradypus tridactylus',
    'orangutan, orang, orangutang, Pongo pygmaeus',
    'gorilla, Gorilla gorilla',
    'chimpanzee, chimp, Pan troglodytes',
    'gibbon, Hylobates lar',
    'siamang, Hylobates syndactylus, Symphalangus syndactylus',
    'guenon, guenon monkey',
    'patas, hussar monkey, Erythrocebus patas',
    'baboon',
    'macaque',
    'langur',
    'colobus, colobus monkey',
    'proboscis monkey, Nasalis larvatus',
    'marmoset',
    'capuchin, ringtail, Cebus capucinus',
    'howler monkey, howler',
    'titi, titi monkey',
    'spider monkey, Ateles geoffroyi',
    'squirrel monkey, Saimiri sciureus',
    'Madagascar cat, ring-tailed lemur, Lemur catta',
    'indri, indris, Indri indri, Indri brevicaudatus',
    'Indian elephant, Elephas maximus',
    'African elephant, Loxodonta africana',
    'lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens',
    'giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca',
    'barracouta, snoek',
    'eel',
    'coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch',
    'rock beauty, Holocanthus tricolor',
    'anemone fish',
    'sturgeon',
    'gar, garfish, garpike, billfish, Lepisosteus osseus',
    'lionfish',
    'puffer, pufferfish, blowfish, globefish',
    'abacus',
    'abaya',
    "academic gown, academic robe, judge's robe",
    'accordion, piano accordion, squeeze box',
    'acoustic guitar',
    'aircraft carrier, carrier, flattop, attack aircraft carrier',
    'airliner',
    'airship, dirigible',
    'altar',
    'ambulance',
    'amphibian, amphibious vehicle',
    'analog clock',
    'apiary, bee house',
    'apron',
    'ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin',
    'assault rifle, assault gun',
    'backpack, back pack, knapsack, packsack, rucksack, haversack',
    'bakery, bakeshop, bakehouse',
    'balance beam, beam',
    'balloon',
    'ballpoint, ballpoint pen, ballpen, Biro',
    'Band Aid',
    'banjo',
    'bannister, banister, balustrade, balusters, handrail',
    'barbell',
    'barber chair',
    'barbershop',
    'barn',
    'barometer',
    'barrel, cask',
    'barrow, garden cart, lawn cart, wheelbarrow',
    'baseball',
    'basketball',
    'bassinet',
    'bassoon',
    'bathing cap, swimming cap',
    'bath towel',
    'bathtub, bathing tub, bath, tub',
    'beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon',
    'beacon, lighthouse, beacon light, pharos',
    'beaker',
    'bearskin, busby, shako',
    'beer bottle',
    'beer glass',
    'bell cote, bell cot',
    'bib',
    'bicycle-built-for-two, tandem bicycle, tandem',
    'bikini, two-piece',
    'binder, ring-binder',
    'binoculars, field glasses, opera glasses',
    'birdhouse',
    'boathouse',
    'bobsled, bobsleigh, bob',
    'bolo tie, bolo, bola tie, bola',
    'bonnet, poke bonnet',
    'bookcase',
    'bookshop, bookstore, bookstall',
    'bottlecap',
    'bow',
    'bow tie, bow-tie, bowtie',
    'brass, memorial tablet, plaque',
    'brassiere, bra, bandeau',
    'breakwater, groin, groyne, mole, bulwark, seawall, jetty',
    'breastplate, aegis, egis',
    'broom',
    'bucket, pail',
    'buckle',
    'bulletproof vest',
    'bullet train, bullet',
    'butcher shop, meat market',
    'cab, hack, taxi, taxicab',
    'caldron, cauldron',
    'candle, taper, wax light',
    'cannon',
    'canoe',
    'can opener, tin opener',
    'cardigan',
    'car mirror',
    'carousel, carrousel, merry-go-round, roundabout, whirligig',
    "carpenter's kit, tool kit",
    'carton',
    'car wheel',
    'cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM',
    'cassette',
    'cassette player',
    'castle',
    'catamaran',
    'CD player',
    'cello, violoncello',
    'cellular telephone, cellular phone, cellphone, cell, mobile phone',
    'chain',
    'chainlink fence',
    'chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour',
    'chain saw, chainsaw',
    'chest',
    'chiffonier, commode',
    'chime, bell, gong',
    'china cabinet, china closet',
    'Christmas stocking',
    'church, church building',
    'cinema, movie theater, movie theatre, movie house, picture palace',
    'cleaver, meat cleaver, chopper',
    'cliff dwelling',
    'cloak',
    'clog, geta, patten, sabot',
    'cocktail shaker',
    'coffee mug',
    'coffeepot',
    'coil, spiral, volute, whorl, helix',
    'combination lock',
    'computer keyboard, keypad',
    'confectionery, confectionary, candy store',
    'container ship, containership, container vessel',
    'convertible',
    'corkscrew, bottle screw',
    'cornet, horn, trumpet, trump',
    'cowboy boot',
    'cowboy hat, ten-gallon hat',
    'cradle',
    'crane',
    'crash helmet',
    'crate',
    'crib, cot',
    'Crock Pot',
    'croquet ball',
    'crutch',
    'cuirass',
    'dam, dike, dyke',
    'desk',
    'desktop computer',
    'dial telephone, dial phone',
    'diaper, nappy, napkin',
    'digital clock',
    'digital watch',
    'dining table, board',
    'dishrag, dishcloth',
    'dishwasher, dish washer, dishwashing machine',
    'disk brake, disc brake',
    'dock, dockage, docking facility',
    'dogsled, dog sled, dog sleigh',
    'dome',
    'doormat, welcome mat',
    'drilling platform, offshore rig',
    'drum, membranophone, tympan',
    'drumstick',
    'dumbbell',
    'Dutch oven',
    'electric fan, blower',
    'electric guitar',
    'electric locomotive',
    'entertainment center',
    'envelope',
    'espresso maker',
    'face powder',
    'feather boa, boa',
    'file, file cabinet, filing cabinet',
    'fireboat',
    'fire engine, fire truck',
    'fire screen, fireguard',
    'flagpole, flagstaff',
    'flute, transverse flute',
    'folding chair',
    'football helmet',
    'forklift',
    'fountain',
    'fountain pen',
    'four-poster',
    'freight car',
    'French horn, horn',
    'frying pan, frypan, skillet',
    'fur coat',
    'garbage truck, dustcart',
    'gasmask, respirator, gas helmet',
    'gas pump, gasoline pump, petrol pump, island dispenser',
    'goblet',
    'go-kart',
    'golf ball',
    'golfcart, golf cart',
    'gondola',
    'gong, tam-tam',
    'gown',
    'grand piano, grand',
    'greenhouse, nursery, glasshouse',
    'grille, radiator grille',
    'grocery store, grocery, food market, market',
    'guillotine',
    'hair slide',
    'hair spray',
    'half track',
    'hammer',
    'hamper',
    'hand blower, blow dryer, blow drier, hair dryer, hair drier',
    'hand-held computer, hand-held microcomputer',
    'handkerchief, hankie, hanky, hankey',
    'hard disc, hard disk, fixed disk',
    'harmonica, mouth organ, harp, mouth harp',
    'harp',
    'harvester, reaper',
    'hatchet',
    'holster',
    'home theater, home theatre',
    'honeycomb',
    'hook, claw',
    'hoopskirt, crinoline',
    'horizontal bar, high bar',
    'horse cart, horse-cart',
    'hourglass',
    'iPod',
    'iron, smoothing iron',
    "jack-o'-lantern",
    'jean, blue jean, denim',
    'jeep, landrover',
    'jersey, T-shirt, tee shirt',
    'jigsaw puzzle',
    'jinrikisha, ricksha, rickshaw',
    'joystick',
    'kimono',
    'knee pad',
    'knot',
    'lab coat, laboratory coat',
    'ladle',
    'lampshade, lamp shade',
    'laptop, laptop computer',
    'lawn mower, mower',
    'lens cap, lens cover',
    'letter opener, paper knife, paperknife',
    'library',
    'lifeboat',
    'lighter, light, igniter, ignitor',
    'limousine, limo',
    'liner, ocean liner',
    'lipstick, lip rouge',
    'Loafer',
    'lotion',
    'loudspeaker, speaker, speaker unit, loudspeaker system, speaker system',
    "loupe, jeweler's loupe",
    'lumbermill, sawmill',
    'magnetic compass',
    'mailbag, postbag',
    'mailbox, letter box',
    'maillot',
    'maillot, tank suit',
    'manhole cover',
    'maraca',
    'marimba, xylophone',
    'mask',
    'matchstick',
    'maypole',
    'maze, labyrinth',
    'measuring cup',
    'medicine chest, medicine cabinet',
    'megalith, megalithic structure',
    'microphone, mike',
    'microwave, microwave oven',
    'military uniform',
    'milk can',
    'minibus',
    'miniskirt, mini',
    'minivan',
    'missile',
    'mitten',
    'mixing bowl',
    'mobile home, manufactured home',
    'Model T',
    'modem',
    'monastery',
    'monitor',
    'moped',
    'mortar',
    'mortarboard',
    'mosque',
    'mosquito net',
    'motor scooter, scooter',
    'mountain bike, all-terrain bike, off-roader',
    'mountain tent',
    'mouse, computer mouse',
    'mousetrap',
    'moving van',
    'muzzle',
    'nail',
    'neck brace',
    'necklace',
    'nipple',
    'notebook, notebook computer',
    'obelisk',
    'oboe, hautboy, hautbois',
    'ocarina, sweet potato',
    'odometer, hodometer, mileometer, milometer',
    'oil filter',
    'organ, pipe organ',
    'oscilloscope, scope, cathode-ray oscilloscope, CRO',
    'overskirt',
    'oxcart',
    'oxygen mask',
    'packet',
    'paddle, boat paddle',
    'paddlewheel, paddle wheel',
    'padlock',
    'paintbrush',
    "pajama, pyjama, pj's, jammies",
    'palace',
    'panpipe, pandean pipe, syrinx',
    'paper towel',
    'parachute, chute',
    'parallel bars, bars',
    'park bench',
    'parking meter',
    'passenger car, coach, carriage',
    'patio, terrace',
    'pay-phone, pay-station',
    'pedestal, plinth, footstall',
    'pencil box, pencil case',
    'pencil sharpener',
    'perfume, essence',
    'Petri dish',
    'photocopier',
    'pick, plectrum, plectron',
    'pickelhaube',
    'picket fence, paling',
    'pickup, pickup truck',
    'pier',
    'piggy bank, penny bank',
    'pill bottle',
    'pillow',
    'ping-pong ball',
    'pinwheel',
    'pirate, pirate ship',
    'pitcher, ewer',
    "plane, carpenter's plane, woodworking plane",
    'planetarium',
    'plastic bag',
    'plate rack',
    'plow, plough',
    "plunger, plumber's helper",
    'Polaroid camera, Polaroid Land camera',
    'pole',
    'police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria',
    'poncho',
    'pool table, billiard table, snooker table',
    'pop bottle, soda bottle',
    'pot, flowerpot',
    "potter's wheel",
    'power drill',
    'prayer rug, prayer mat',
    'printer',
    'prison, prison house',
    'projectile, missile',
    'projector',
    'puck, hockey puck',
    'punching bag, punch bag, punching ball, punchball',
    'purse',
    'quill, quill pen',
    'quilt, comforter, comfort, puff',
    'racer, race car, racing car',
    'racket, racquet',
    'radiator',
    'radio, wireless',
    'radio telescope, radio reflector',
    'rain barrel',
    'recreational vehicle, RV, R.V.',
    'reel',
    'reflex camera',
    'refrigerator, icebox',
    'remote control, remote',
    'restaurant, eating house, eating place, eatery',
    'revolver, six-gun, six-shooter',
    'rifle',
    'rocking chair, rocker',
    'rotisserie',
    'rubber eraser, rubber, pencil eraser',
    'rugby ball',
    'rule, ruler',
    'running shoe',
    'safe',
    'safety pin',
    'saltshaker, salt shaker',
    'sandal',
    'sarong',
    'sax, saxophone',
    'scabbard',
    'scale, weighing machine',
    'school bus',
    'schooner',
    'scoreboard',
    'screen, CRT screen',
    'screw',
    'screwdriver',
    'seat belt, seatbelt',
    'sewing machine',
    'shield, buckler',
    'shoe shop, shoe-shop, shoe store',
    'shoji',
    'shopping basket',
    'shopping cart',
    'shovel',
    'shower cap',
    'shower curtain',
    'ski',
    'ski mask',
    'sleeping bag',
    'slide rule, slipstick',
    'sliding door',
    'slot, one-armed bandit',
    'snorkel',
    'snowmobile',
    'snowplow, snowplough',
    'soap dispenser',
    'soccer ball',
    'sock',
    'solar dish, solar collector, solar furnace',
    'sombrero',
    'soup bowl',
    'space bar',
    'space heater',
    'space shuttle',
    'spatula',
    'speedboat',
    "spider web, spider's web",
    'spindle',
    'sports car, sport car',
    'spotlight, spot',
    'stage',
    'steam locomotive',
    'steel arch bridge',
    'steel drum',
    'stethoscope',
    'stole',
    'stone wall',
    'stopwatch, stop watch',
    'stove',
    'strainer',
    'streetcar, tram, tramcar, trolley, trolley car',
    'stretcher',
    'studio couch, day bed',
    'stupa, tope',
    'submarine, pigboat, sub, U-boat',
    'suit, suit of clothes',
    'sundial',
    'sunglass',
    'sunglasses, dark glasses, shades',
    'sunscreen, sunblock, sun blocker',
    'suspension bridge',
    'swab, swob, mop',
    'sweatshirt',
    'swimming trunks, bathing trunks',
    'swing',
    'switch, electric switch, electrical switch',
    'syringe',
    'table lamp',
    'tank, army tank, armored combat vehicle, armoured combat vehicle',
    'tape player',
    'teapot',
    'teddy, teddy bear',
    'television, television system',
    'tennis ball',
    'thatch, thatched roof',
    'theater curtain, theatre curtain',
    'thimble',
    'thresher, thrasher, threshing machine',
    'throne',
    'tile roof',
    'toaster',
    'tobacco shop, tobacconist shop, tobacconist',
    'toilet seat',
    'torch',
    'totem pole',
    'tow truck, tow car, wrecker',
    'toyshop',
    'tractor',
    'trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi',
    'tray',
    'trench coat',
    'tricycle, trike, velocipede',
    'trimaran',
    'tripod',
    'triumphal arch',
    'trolleybus, trolley coach, trackless trolley',
    'trombone',
    'tub, vat',
    'turnstile',
    'typewriter keyboard',
    'umbrella',
    'unicycle, monocycle',
    'upright, upright piano',
    'vacuum, vacuum cleaner',
    'vase',
    'vault',
    'velvet',
    'vending machine',
    'vestment',
    'viaduct',
    'violin, fiddle',
    'volleyball',
    'waffle iron',
    'wall clock',
    'wallet, billfold, notecase, pocketbook',
    'wardrobe, closet, press',
    'warplane, military plane',
    'washbasin, handbasin, washbowl, lavabo, wash-hand basin',
    'washer, automatic washer, washing machine',
    'water bottle',
    'water jug',
    'water tower',
    'whiskey jug',
    'whistle',
    'wig',
    'window screen',
    'window shade',
    'Windsor tie',
    'wine bottle',
    'wing',
    'wok',
    'wooden spoon',
    'wool, woolen, woollen',
    'worm fence, snake fence, snake-rail fence, Virginia fence',
    'wreck',
    'yawl',
    'yurt',
    'web site, website, internet site, site',
    'comic book',
    'crossword puzzle, crossword',
    'street sign',
    'traffic light, traffic signal, stoplight',
    'book jacket, dust cover, dust jacket, dust wrapper',
    'menu',
    'plate',
    'guacamole',
    'consomme',
    'hot pot, hotpot',
    'trifle',
    'ice cream, icecream',
    'ice lolly, lolly, lollipop, popsicle',
    'French loaf',
    'bagel, beigel',
    'pretzel',
    'cheeseburger',
    'hotdog, hot dog, red hot',
    'mashed potato',
    'head cabbage',
    'broccoli',
    'cauliflower',
    'zucchini, courgette',
    'spaghetti squash',
    'acorn squash',
    'butternut squash',
    'cucumber, cuke',
    'artichoke, globe artichoke',
    'bell pepper',
    'cardoon',
    'mushroom',
    'Granny Smith',
    'strawberry',
    'orange',
    'lemon',
    'fig',
    'pineapple, ananas',
    'banana',
    'jackfruit, jak, jack',
    'custard apple',
    'pomegranate',
    'hay',
    'carbonara',
    'chocolate sauce, chocolate syrup',
    'dough',
    'meat loaf, meatloaf',
    'pizza, pizza pie',
    'potpie',
    'burrito',
    'red wine',
    'espresso',
    'cup',
    'eggnog',
    'alp',
    'bubble',
    'cliff, drop, drop-off',
    'coral reef',
    'geyser',
    'lakeside, lakeshore',
    'promontory, headland, head, foreland',
    'sandbar, sand bar',
    'seashore, coast, seacoast, sea-coast',
    'valley, vale',
    'volcano',
    'ballplayer, baseball player',
    'groom, bridegroom',
    'scuba diver',
    'rapeseed',
    'daisy',
    "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum",
    'corn',
    'acorn',
    'hip, rose hip, rosehip',
    'buckeye, horse chestnut, conker',
    'coral fungus',
    'agaric',
    'gyromitra',
    'stinkhorn, carrion fungus',
    'earthstar',
    'hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa',
    'bolete',
    'ear, spike, capitulum',
    'toilet tissue, toilet paper, bathroom tissue'
]

def label_to_name(label):
    global _lut
    return _lut[label]


================================================
FILE: tools/inception_v3_imagenet.py
================================================
from tools.utils import optimistic_restore
import tensorflow as tf
import tensorflow.contrib.slim as slim
import tensorflow.contrib.slim.nets as nets
import functools
import os

SIZE = 299

# to make this work, you need to download:
# http://download.tensorflow.org/models/inception_v3_2016_08_28.tar.gz
# and decompress it in the `data` directory

_INCEPTION_CHECKPOINT_NAME = 'inception_v3.ckpt'
INCEPTION_CHECKPOINT_PATH = os.path.join(
    os.path.dirname(__file__),
    'data',
    _INCEPTION_CHECKPOINT_NAME
)

def _get_model(reuse):
    arg_scope = nets.inception.inception_v3_arg_scope(weight_decay=0.0)
    func = nets.inception.inception_v3
    @functools.wraps(func)
    def network_fn(images):
        with slim.arg_scope(arg_scope):
            return func(images, 1001, is_training=False, reuse=reuse)
    if hasattr(func, 'default_image_size'):
        network_fn.default_image_size = func.default_image_size
    return network_fn

def _preprocess(image, height, width, scope=None):
    with tf.name_scope(scope, 'eval_image', [image, height, width]):
        if image.dtype != tf.float32:
            image = tf.image.convert_image_dtype(image, dtype=tf.float32)
        image = tf.image.resize_bilinear(image, [height, width], align_corners=False)
        image = tf.subtract(image, 0.5)
        image = tf.multiply(image, 2.0)
        return image

# input is [batch, 256, 256, 3], pixels in [0, 1]
# output is [batch, 10]
_inception_initialized = False
def model(sess, image):
    global _inception_initialized
    network_fn = _get_model(reuse=_inception_initialized)
    size = network_fn.default_image_size
    preprocessed = _preprocess(image, size, size)
    logits, _ = network_fn(preprocessed)
    logits = logits[:,1:] # ignore background class
    predictions = tf.argmax(logits, 1)

    if not _inception_initialized:
        optimistic_restore(sess, INCEPTION_CHECKPOINT_PATH)
        _inception_initialized = True

    return logits, predictions


================================================
FILE: tools/logging_utils.py
================================================
from matplotlib import pyplot as plt
from tools.utils import *
import os
import numpy as np
import scipy

def log_output(sess, eval_logits, eval_preds, x, adv, initial_img, \
        target_class, out_dir, orig_class, num_queries):
    """
    Evaluate and save a bunch of metadata about the optimization trajectory and
    the final adversarial examples generated.
    """
    eval_logits_, eval_preds_ = sess.run([eval_logits, eval_preds], {x: adv})
    eval_logits_orig_, eval_preds_orig_ = sess.run([eval_logits, eval_preds], {x: initial_img})
    eval_dir = os.path.join(out_dir, 'eval')
    if not os.path.exists(eval_dir):
        os.makedirs(eval_dir)

    with open(os.path.join(eval_dir, 'eval.txt'), 'w') as fout:
        fout.write('true %d\n' % orig_class)
        fout.write('target %d\n' % target_class)
        fout.write('queries %d\n' % num_queries)
    scipy.misc.imsave(os.path.join(eval_dir, 'original.png'), initial_img)
    np.save(os.path.join(eval_dir, 'original.npy'), initial_img)
    scipy.misc.imsave(os.path.join(eval_dir, 'adversarial.png'), adv)
    np.save(os.path.join(eval_dir, 'adversarial.npy'), adv)
    with open(os.path.join(eval_dir, 'sample.txt'), 'w') as fout:
        fout.write('orig_pred %d\n' % eval_preds_orig_[0])
        orig_p = softmax(eval_logits_orig_[0])
        fout.write('orig_conf %.5f\n' % np.max(orig_p))
        fout.write('orig_true_conf %.5f\n' % orig_p[orig_class])
        fout.write('orig_adv_conf %.5f\n' % orig_p[target_class])
        fout.write('adv_pred %d\n' % eval_preds_[0])
        adv_p = softmax(eval_logits_[0])
        fout.write('adv_conf %.5f\n' % np.max(adv_p))
        fout.write('adv_true_conf %.5f\n' % adv_p[orig_class])
        fout.write('adv_adv_conf %.5f\n' % adv_p[target_class])


def render_frame(sess, image, save_index, render_logits, render_feed, out_dir):
    # actually draw the figure
    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 8))
    # image
    ax1.imshow(image)
    fig.sca(ax1)
    plt.xticks([])
    plt.yticks([])
    # classifications
    probs = softmax(sess.run(render_logits, {render_feed: image})[0])
    topk = probs.argsort()[-5:][::-1]
    topprobs = probs[topk]
    barlist = ax2.bar(range(5), topprobs)
    for i, v in enumerate(topk):
        if v == orig_class:
            barlist[i].set_color('g')
        if v == target_class:
            barlist[i].set_color('r')
    plt.sca(ax2)
    plt.ylim([0, 1.1])
    plt.xticks(range(5), [label_to_name(i)[:15] for i in topk], rotation='vertical')
    fig.subplots_adjust(bottom=0.2)
    path = os.path.join(out_dir, 'frame%06d.png' % save_index)
    if os.path.exists(path):
        os.remove(path)
    plt.savefig(path)
    plt.close()


================================================
FILE: tools/utils.py
================================================
import tensorflow as tf
import PIL
from tensorflow.python.framework import ops
import numpy as np
import os
import pickle
from matplotlib import pyplot as plt

def image_of_class(y, imagenet_path=None):
    """
    Gets an image of a prespecified class. To save computation time we use a
    presaved dictionary of an index for each class, but an attacker can also
    randomize this, or search for the best starting images.
    """
    im_indices = pickle.load(open("tools/data/imagenet.pickle", "rb"))
    return get_image(im_indices[y], imagenet_path)[0].copy()

def pseudorandom_target(index, total_indices, true_class):
    rng = np.random.RandomState(index)
    target = true_class
    while target == true_class:
        target = rng.randint(0, total_indices)
    return target

def pseudorandom_target_image(orig_index, total_indices):
    rng = np.random.RandomState(orig_index)
    target_img_index = orig_index
    while target_img_index == orig_index:
        target_img_index = rng.randint(0, total_indices)
    return target_img_index

# get center crop
def load_image(path):
    image = PIL.Image.open(path)
    if image.height > image.width:
        height_off = int((image.height - image.width)/2)
        image = image.crop((0, height_off, image.width, height_off+image.width))
    elif image.width > image.height:
        width_off = int((image.width - image.height)/2)
        image = image.crop((width_off, 0, width_off+image.height, image.height))
    image = image.resize((299, 299))
    img = np.asarray(image).astype(np.float32) / 255.0
    if img.ndim == 2:
        img = np.repeat(img[:,:,np.newaxis], repeats=3, axis=2)
    if img.shape[2] == 4:
        # alpha channel
        img = img[:,:,:3]
    return img

def get_image(index, imagenet_path=None):
    data_path = os.path.join(imagenet_path, 'val')
    image_paths = sorted([os.path.join(data_path, i) for i in os.listdir(data_path)])
    assert len(image_paths) == 50000
    labels_path = os.path.join(imagenet_path, 'val.txt')
    with open(labels_path) as labels_file:
        labels = [i.split(' ') for i in labels_file.read().strip().split('\n')]
        labels = {os.path.basename(i[0]): int(i[1]) for i in labels}
    def get(index):
        path = image_paths[index]
        x = load_image(path)
        y = labels[os.path.basename(path)]
        return x, y
    return get(index)

def one_hot(index, total):
    arr = np.zeros((total))
    arr[index] = 1.0
    return arr

def optimistic_restore(session, save_file):
    reader = tf.train.NewCheckpointReader(save_file)
    saved_shapes = reader.get_variable_to_shape_map()
    var_names = sorted([(var.name, var.name.split(':')[0]) for var in tf.global_variables()
            if var.name.split(':')[0] in saved_shapes])
    restore_vars = []
    with tf.variable_scope('', reuse=True):
        for var_name, saved_var_name in var_names:
            curr_var = tf.get_variable(saved_var_name)
            var_shape = curr_var.get_shape().as_list()
            if var_shape == saved_shapes[saved_var_name]:
                restore_vars.append(curr_var)
    saver = tf.train.Saver(restore_vars)
    saver.restore(session, save_file)

def softmax(x):
    e_x = np.exp(x - np.max(x))
    return e_x / e_x.sum(axis=0)

def int_shape(tensor):
    return list(map(int, tensor.get_shape()))

def chunks(l, n):
    for i in range(0, len(l), n):
        yield l[i:i + n]

def hms(seconds):
    seconds = int(seconds)
    hours = (seconds // (60 * 60))
    minutes = (seconds // 60) % 60
    seconds = seconds % 60
    if hours > 0:
        return '%d hrs %d min' % (hours, minutes)
    elif minutes > 0:
        return '%d min %d sec' % (minutes, seconds)
    else:
        return '%d sec' % seconds

_py_func_id = 0
def py_func(func, inp, Tout, stateful=True, name=None, grad=None):
    global _py_func_id

    rnd_name = 'PyFuncGrad' + '%08d' % _py_func_id
    _py_func_id += 1

    tf.RegisterGradient(rnd_name)(grad)
    g = tf.get_default_graph()
    with g.gradient_override_map({"PyFunc": rnd_name, "PyFuncStateless": rnd_name}):
        return tf.py_func(func, inp, Tout, stateful=stateful, name=name)

def grad_clip_by_norm(x, clip_norm=1, name=None):
    if isinstance(clip_norm, int):
        clip_norm = float(clip_norm)
    with ops.name_scope(name, "grad_clip_by_norm", [x, clip_norm]) as name:
        identity, = py_func(
            lambda t,
            _: t,
            [x, clip_norm],
            [tf.float32],
            name=name,
            grad=_grad_clip_by_norm_grad,
            stateful=False
        )
        identity.set_shape(x.get_shape())
        return identity

# Actual gradient:
def _grad_clip_by_norm_grad(op, grad):
    _, norm = op.inputs
    return (tf.clip_by_norm(grad, norm), None)

def grad_clip_by_value(x, clip_magnitude=1, name=None):
    if isinstance(clip_magnitude, int):
        clip_magnitude = float(clip_magnitude)
    with ops.name_scope(name, "grad_clip_by_value", [x, clip_magnitude]) as name:
        identity, = py_func(
            lambda t,
            _: t,
            [x, clip_magnitude],
            [tf.float32],
            name=name,
            grad=_grad_clip_by_value_grad,
            stateful=False
        )
        identity.set_shape(x.get_shape())
        return identity

# Actual gradient:
def _grad_clip_by_value_grad(op, grad):
    _, mag = op.inputs
    return (tf.clip_by_value(grad, -mag, mag), None)