[ { "path": "README.md", "content": "# Indoor-segmentation\n## Introduction\n This is an implementation of TensorFlow-based (TF1) DeepLab-ResNet for Indoor-scene segmentation. The provided model is trained on the [ade20k](http://sceneparsing.csail.mit.edu/) dataset. The code is inherited from [tensorflow-deeplab-resnet](https://github.com/DrSleep/tensorflow-deeplab-resnet) by [Drsleep](https://drsleep.github.io/). Since this model is for `robot navigating`, we `re-label 150 classes into 27 classes` in order to easily classify obstacles and road. \n\n### Re-label list: \n```\n1 (wall) <- 9(window), 15(door), 33(fence), 43(pillar), 44(sign board), 145(bullertin board)\n4 (floor) <- 7(road), 14(ground, 30(field), 53(path), 55(runway)\n5 (tree) <- 18(plant)\n8 (furniture) <- 8(bed), 11(cabinet), 14(sofa), 16(table), 19(curtain), 20(chair), 25(shelf), 34(desk) \n7 (stairs) <- 54(stairs)\n26(others) <- class number larger than 26\n```\n\n \n## Quick Start \n### Install dependency \nThe codes are test on `Python 3.7`. Please run the following script to install the packages.\n```bash\npip install -r requirements.txt\n```\n\n### Download pretrained model\nRun the following script to download the provided pretrained model from Google Drive.\n```bash\n./download_models.sh\n```\nOr directly get the pretrained model from [Google Drive](https://drive.google.com/file/d/1o7QrlNxH6BX6uYatlR06-A_cutWD9sNg/view?usp=sharing).\n\n### Demo\nRun the following sample command for inference\n```\npython inference.py --img_path input/IMG_0416_640x480.png --restore_from=pretrained_models/ResNet101/\n```\n\n## Result\n### Video\n[![Demo video](https://img.youtube.com/vi/4OqW3M-eqaQ/0.jpg)](https://youtu.be/4OqW3M-eqaQ)\n### Image\nInput image | Output image\n:-------------------------:|:-------------------------:\n![](https://github.com/hellochick/Indoor-segmentation/blob/master/input/IMG_0416_640x480.png) | ![](https://github.com/hellochick/Indoor-segmentation/blob/master/output/IMG_0416_640x480.png)\n![](https://github.com/hellochick/Indoor-segmentation/blob/master/input/IMG_0417_640x480.png) | ![](https://github.com/hellochick/Indoor-segmentation/blob/master/output/IMG_0417_640x480.png)\n![](https://github.com/hellochick/Indoor-segmentation/blob/master/input/IMG_0418_640x480.png) | ![](https://github.com/hellochick/Indoor-segmentation/blob/master/output/IMG_0418_640x480.png)\n" }, { "path": "download_models.sh", "content": "gdown --fuzzy 1o7QrlNxH6BX6uYatlR06-A_cutWD9sNg\nunzip pretrained_models.zip\n\n" }, { "path": "inference.py", "content": "from __future__ import print_function\n\nimport argparse\nimport os\nimport sys\nimport time\nimport scipy.io as sio\nfrom PIL import Image\n\nimport tensorflow as tf\nimport numpy as np\n\nfrom model import DeepLabResNetModel\n\nIMG_MEAN = np.array((104.00698793,116.66876762,122.67891434), dtype=np.float32)\n\nNUM_CLASSES = 27\nSAVE_DIR = './output/'\nRESTORE_PATH = './restore_weights/'\nmatfn = 'color150.mat'\n\ndef get_arguments():\n parser = argparse.ArgumentParser(description=\"Indoor segmentation parser.\")\n parser.add_argument(\"--img_path\", type=str, default='',\n help=\"Path to the RGB image file.\")\n parser.add_argument(\"--restore_from\", type=str, default=RESTORE_PATH,\n help=\"checkpoint location\")\n\n return parser.parse_args()\n\ndef read_labelcolours(matfn):\n mat = sio.loadmat(matfn)\n color_table = mat['colors']\n shape = color_table.shape\n color_list = [tuple(color_table[i]) for i in range(shape[0])]\n\n return color_list\n\ndef decode_labels(mask, num_images=1, num_classes=150):\n label_colours = read_labelcolours(matfn)\n\n n, h, w, c = mask.shape\n assert(n >= num_images), 'Batch size %d should be greater or equal than number of images to save %d.' % (n, num_images)\n outputs = np.zeros((num_images, h, w, 3), dtype=np.uint8)\n for i in range(num_images):\n img = Image.new('RGB', (len(mask[i, 0]), len(mask[i])))\n pixels = img.load()\n for j_, j in enumerate(mask[i, :, :, 0]):\n for k_, k in enumerate(j):\n if k < num_classes:\n pixels[k_,j_] = label_colours[k]\n outputs[i] = np.array(img)\n return outputs\n\ndef load(saver, sess, ckpt_path):\n saver.restore(sess, ckpt_path)\n print(\"Restored model parameters from {}\".format(ckpt_path))\n\ndef main():\n args = get_arguments()\n filename = args.img_path.split('/')[-1]\n file_type = filename.split('.')[-1]\n\n if os.path.isfile(args.img_path):\n print('successful load img: {0}'.format(args.img_path))\n else:\n print('not found file: {0}'.format(args.img_path))\n sys.exit(0)\n\n # Prepare image.\n if file_type.lower() == 'png':\n img = tf.image.decode_png(tf.read_file(args.img_path), channels=3)\n elif file_type.lower() == 'jpg':\n img = tf.image.decode_jpeg(tf.read_file(args.img_path), channels=3)\n else:\n print('cannot process {0} file.'.format(file_type))\n\n # Convert RGB to BGR.\n img_r, img_g, img_b = tf.split(axis=2, num_or_size_splits=3, value=img)\n img = tf.cast(tf.concat(axis=2, values=[img_b, img_g, img_r]), dtype=tf.float32)\n # Extract mean.\n img -= IMG_MEAN\n\n # Create network.\n net = DeepLabResNetModel({'data': tf.expand_dims(img, dim=0)}, is_training=False, num_classes=NUM_CLASSES)\n\n # Which variables to load.\n restore_var = tf.global_variables()\n\n # Predictions.\n raw_output = net.layers['fc_out']\n raw_output_up = tf.image.resize_bilinear(raw_output, tf.shape(img)[0:2,])\n raw_output_up = tf.argmax(raw_output_up, dimension=3)\n pred = tf.expand_dims(raw_output_up, dim=3)\n\n # Set up TF session and initialize variables.\n config = tf.ConfigProto()\n config.gpu_options.allow_growth = True\n sess = tf.Session(config=config)\n init = tf.global_variables_initializer()\n\n sess.run(init)\n\n # Load weights.\n ckpt = tf.train.get_checkpoint_state(args.restore_from)\n\n if ckpt and ckpt.model_checkpoint_path:\n loader = tf.train.Saver(var_list=restore_var)\n load_step = int(os.path.basename(ckpt.model_checkpoint_path).split('-')[1])\n load(loader, sess, ckpt.model_checkpoint_path)\n else:\n print('No checkpoint file found.')\n load_step = 0\n\n # Perform inference.\n preds = sess.run(pred)\n\n msk = decode_labels(preds, num_classes=NUM_CLASSES)\n im = Image.fromarray(msk[0])\n if not os.path.exists(SAVE_DIR):\n os.makedirs(SAVE_DIR)\n im.save(SAVE_DIR + filename)\n\n print('The output file has been saved to {0}'.format(SAVE_DIR + filename))\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "model.py", "content": "# Converted to TensorFlow .caffemodel\n# with the DeepLab-ResNet configuration.\n# The batch normalisation layer is provided by\n# the slim library (https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/slim).\n\nfrom network import Network\nimport tensorflow as tf\n\nclass DeepLabResNetModel(Network):\n def setup(self, is_training, num_classes):\n '''Network definition.\n\n Args:\n is_training: whether to update the running mean and variance of the batch normalisation layer.\n If the batch size is small, it is better to keep the running mean and variance of\n the-pretrained model frozen.\n num_classes: number of classes to predict (including background).\n '''\n (self.feed('data')\n .conv(7, 7, 64, 2, 2, biased=False, relu=False, name='conv1')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn_conv1')\n .max_pool(3, 3, 2, 2, name='pool1')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res2a_branch1')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn2a_branch1'))\n\n (self.feed('pool1')\n .conv(1, 1, 64, 1, 1, biased=False, relu=False, name='res2a_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn2a_branch2a')\n .conv(3, 3, 64, 1, 1, biased=False, relu=False, name='res2a_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn2a_branch2b')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res2a_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn2a_branch2c'))\n\n (self.feed('bn2a_branch1',\n 'bn2a_branch2c')\n .add(name='res2a')\n .relu(name='res2a_relu')\n .conv(1, 1, 64, 1, 1, biased=False, relu=False, name='res2b_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn2b_branch2a')\n .conv(3, 3, 64, 1, 1, biased=False, relu=False, name='res2b_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn2b_branch2b')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res2b_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn2b_branch2c'))\n\n (self.feed('res2a_relu',\n 'bn2b_branch2c')\n .add(name='res2b')\n .relu(name='res2b_relu')\n .conv(1, 1, 64, 1, 1, biased=False, relu=False, name='res2c_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn2c_branch2a')\n .conv(3, 3, 64, 1, 1, biased=False, relu=False, name='res2c_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn2c_branch2b')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res2c_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn2c_branch2c'))\n\n (self.feed('res2b_relu',\n 'bn2c_branch2c')\n .add(name='res2c')\n .relu(name='res2c_relu')\n .conv(1, 1, 512, 2, 2, biased=False, relu=False, name='res3a_branch1')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn3a_branch1'))\n\n (self.feed('res2c_relu')\n .conv(1, 1, 128, 2, 2, biased=False, relu=False, name='res3a_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn3a_branch2a')\n .conv(3, 3, 128, 1, 1, biased=False, relu=False, name='res3a_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn3a_branch2b')\n .conv(1, 1, 512, 1, 1, biased=False, relu=False, name='res3a_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn3a_branch2c'))\n\n (self.feed('bn3a_branch1',\n 'bn3a_branch2c')\n .add(name='res3a')\n .relu(name='res3a_relu')\n .conv(1, 1, 128, 1, 1, biased=False, relu=False, name='res3b1_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn3b1_branch2a')\n .conv(3, 3, 128, 1, 1, biased=False, relu=False, name='res3b1_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn3b1_branch2b')\n .conv(1, 1, 512, 1, 1, biased=False, relu=False, name='res3b1_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn3b1_branch2c'))\n\n (self.feed('res3a_relu',\n 'bn3b1_branch2c')\n .add(name='res3b1')\n .relu(name='res3b1_relu')\n .conv(1, 1, 128, 1, 1, biased=False, relu=False, name='res3b2_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn3b2_branch2a')\n .conv(3, 3, 128, 1, 1, biased=False, relu=False, name='res3b2_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn3b2_branch2b')\n .conv(1, 1, 512, 1, 1, biased=False, relu=False, name='res3b2_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn3b2_branch2c'))\n\n (self.feed('res3b1_relu',\n 'bn3b2_branch2c')\n .add(name='res3b2')\n .relu(name='res3b2_relu')\n .conv(1, 1, 128, 1, 1, biased=False, relu=False, name='res3b3_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn3b3_branch2a')\n .conv(3, 3, 128, 1, 1, biased=False, relu=False, name='res3b3_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn3b3_branch2b')\n .conv(1, 1, 512, 1, 1, biased=False, relu=False, name='res3b3_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn3b3_branch2c'))\n\n (self.feed('res3b2_relu',\n 'bn3b3_branch2c')\n .add(name='res3b3')\n .relu(name='res3b3_relu')\n .conv(1, 1, 1024, 2, 2, biased=False, relu=False, name='res4a_branch1')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn4a_branch1'))\n\n ### block 4\n (self.feed('res3b3_relu')\n .conv(1, 1, 256, 2, 2, biased=False, relu=False, name='res4a_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4a_branch2a')\n .atrous_conv(3, 3, 256, 1, padding='SAME', biased=False, relu=False, name='res4a_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4a_branch2b')\n .conv(1, 1, 1024, 1, 1, biased=False, relu=False, name='res4a_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn4a_branch2c'))\n\n (self.feed('bn4a_branch1',\n 'bn4a_branch2c')\n .add(name='res4a')\n .relu(name='res4a_relu')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res4b1_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b1_branch2a')\n .atrous_conv(3, 3, 256, 4, padding='SAME', biased=False, relu=False, name='res4b1_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b1_branch2b')\n .conv(1, 1, 1024, 1, 1, biased=False, relu=False, name='res4b1_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn4b1_branch2c'))\n\n (self.feed('res4a_relu',\n 'bn4b1_branch2c')\n .add(name='res4b1')\n .relu(name='res4b1_relu')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res4b2_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b2_branch2a')\n .atrous_conv(3, 3, 256, 2, padding='SAME', biased=False, relu=False, name='res4b2_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b2_branch2b')\n .conv(1, 1, 1024, 1, 1, biased=False, relu=False, name='res4b2_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn4b2_branch2c'))\n\n ### block 4\n\n ### block 5\n (self.feed('res4b1_relu',\n 'bn4b2_branch2c')\n .add(name='res4b2')\n .relu(name='res4b2_relu')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res4b3_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b3_branch2a')\n .atrous_conv(3, 3, 256, 2, padding='SAME', biased=False, relu=False, name='res4b3_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b3_branch2b')\n .conv(1, 1, 1024, 1, 1, biased=False, relu=False, name='res4b3_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn4b3_branch2c'))\n\n (self.feed('res4b2_relu',\n 'bn4b3_branch2c')\n .add(name='res4b3')\n .relu(name='res4b3_relu')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res4b4_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b4_branch2a')\n .atrous_conv(3, 3, 256, 4, padding='SAME', biased=False, relu=False, name='res4b4_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b4_branch2b')\n .conv(1, 1, 1024, 1, 1, biased=False, relu=False, name='res4b4_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn4b4_branch2c'))\n\n (self.feed('res4b3_relu',\n 'bn4b4_branch2c')\n .add(name='res4b4')\n .relu(name='res4b4_relu')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res4b5_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b5_branch2a')\n .atrous_conv(3, 3, 256, 2, padding='SAME', biased=False, relu=False, name='res4b5_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b5_branch2b')\n .conv(1, 1, 1024, 1, 1, biased=False, relu=False, name='res4b5_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn4b5_branch2c'))\n\n ### block 5\n\n ### block 6\n (self.feed('res4b4_relu',\n 'bn4b5_branch2c')\n .add(name='res4b5')\n .relu(name='res4b5_relu')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res4b6_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b6_branch2a')\n .atrous_conv(3, 3, 256, 2, padding='SAME', biased=False, relu=False, name='res4b6_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b6_branch2b')\n .conv(1, 1, 1024, 1, 1, biased=False, relu=False, name='res4b6_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn4b6_branch2c'))\n\n (self.feed('res4b5_relu',\n 'bn4b6_branch2c')\n .add(name='res4b6')\n .relu(name='res4b6_relu')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res4b7_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b7_branch2a')\n .atrous_conv(3, 3, 256, 4, padding='SAME', biased=False, relu=False, name='res4b7_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b7_branch2b')\n .conv(1, 1, 1024, 1, 1, biased=False, relu=False, name='res4b7_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn4b7_branch2c'))\n\n (self.feed('res4b6_relu',\n 'bn4b7_branch2c')\n .add(name='res4b7')\n .relu(name='res4b7_relu')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res4b8_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b8_branch2a')\n .atrous_conv(3, 3, 256, 2, padding='SAME', biased=False, relu=False, name='res4b8_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b8_branch2b')\n .conv(1, 1, 1024, 1, 1, biased=False, relu=False, name='res4b8_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn4b8_branch2c'))\n\n ### block 6\n\n ### block 7\n (self.feed('res4b7_relu',\n 'bn4b8_branch2c')\n .add(name='res4b8')\n .relu(name='res4b8_relu')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res4b9_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b9_branch2a')\n .atrous_conv(3, 3, 256, 2, padding='SAME', biased=False, relu=False, name='res4b9_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b9_branch2b')\n .conv(1, 1, 1024, 1, 1, biased=False, relu=False, name='res4b9_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn4b9_branch2c'))\n\n (self.feed('res4b8_relu',\n 'bn4b9_branch2c')\n .add(name='res4b9')\n .relu(name='res4b9_relu')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res4b10_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b10_branch2a')\n .atrous_conv(3, 3, 256, 4, padding='SAME', biased=False, relu=False, name='res4b10_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b10_branch2b')\n .conv(1, 1, 1024, 1, 1, biased=False, relu=False, name='res4b10_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn4b10_branch2c'))\n\n (self.feed('res4b9_relu',\n 'bn4b10_branch2c')\n .add(name='res4b10')\n .relu(name='res4b10_relu')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res4b11_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b11_branch2a')\n .atrous_conv(3, 3, 256, 2, padding='SAME', biased=False, relu=False, name='res4b11_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b11_branch2b')\n .conv(1, 1, 1024, 1, 1, biased=False, relu=False, name='res4b11_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn4b11_branch2c'))\n\n ### block 7\n (self.feed('res4b10_relu',\n 'bn4b11_branch2c')\n .add(name='res4b11')\n .relu(name='res4b11_relu')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res4b12_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b12_branch2a')\n .atrous_conv(3, 3, 256, 2, padding='SAME', biased=False, relu=False, name='res4b12_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b12_branch2b')\n .conv(1, 1, 1024, 1, 1, biased=False, relu=False, name='res4b12_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn4b12_branch2c'))\n\n (self.feed('res4b11_relu',\n 'bn4b12_branch2c')\n .add(name='res4b12')\n .relu(name='res4b12_relu')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res4b13_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b13_branch2a')\n .atrous_conv(3, 3, 256, 4, padding='SAME', biased=False, relu=False, name='res4b13_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b13_branch2b')\n .conv(1, 1, 1024, 1, 1, biased=False, relu=False, name='res4b13_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn4b13_branch2c'))\n\n (self.feed('res4b12_relu',\n 'bn4b13_branch2c')\n .add(name='res4b13')\n .relu(name='res4b13_relu')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res4b14_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b14_branch2a')\n .atrous_conv(3, 3, 256, 2, padding='SAME', biased=False, relu=False, name='res4b14_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b14_branch2b')\n .conv(1, 1, 1024, 1, 1, biased=False, relu=False, name='res4b14_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn4b14_branch2c'))\n\n (self.feed('res4b13_relu',\n 'bn4b14_branch2c')\n .add(name='res4b14')\n .relu(name='res4b14_relu')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res4b15_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b15_branch2a')\n .atrous_conv(3, 3, 256, 2, padding='SAME', biased=False, relu=False, name='res4b15_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b15_branch2b')\n .conv(1, 1, 1024, 1, 1, biased=False, relu=False, name='res4b15_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn4b15_branch2c'))\n\n (self.feed('res4b14_relu',\n 'bn4b15_branch2c')\n .add(name='res4b15')\n .relu(name='res4b15_relu')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res4b16_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b16_branch2a')\n .atrous_conv(3, 3, 256, 4, padding='SAME', biased=False, relu=False, name='res4b16_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b16_branch2b')\n .conv(1, 1, 1024, 1, 1, biased=False, relu=False, name='res4b16_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn4b16_branch2c'))\n\n (self.feed('res4b15_relu',\n 'bn4b16_branch2c')\n .add(name='res4b16')\n .relu(name='res4b16_relu')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res4b17_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b17_branch2a')\n .atrous_conv(3, 3, 256, 2, padding='SAME', biased=False, relu=False, name='res4b17_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b17_branch2b')\n .conv(1, 1, 1024, 1, 1, biased=False, relu=False, name='res4b17_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn4b17_branch2c'))\n\n (self.feed('res4b16_relu',\n 'bn4b17_branch2c')\n .add(name='res4b17')\n .relu(name='res4b17_relu')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res4b18_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b18_branch2a')\n .atrous_conv(3, 3, 256, 2, padding='SAME', biased=False, relu=False, name='res4b18_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b18_branch2b')\n .conv(1, 1, 1024, 1, 1, biased=False, relu=False, name='res4b18_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn4b18_branch2c'))\n\n (self.feed('res4b17_relu',\n 'bn4b18_branch2c')\n .add(name='res4b18')\n .relu(name='res4b18_relu')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res4b19_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b19_branch2a')\n .atrous_conv(3, 3, 256, 4, padding='SAME', biased=False, relu=False, name='res4b19_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b19_branch2b')\n .conv(1, 1, 1024, 1, 1, biased=False, relu=False, name='res4b19_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn4b19_branch2c'))\n\n (self.feed('res4b18_relu',\n 'bn4b19_branch2c')\n .add(name='res4b19')\n .relu(name='res4b19_relu')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res4b20_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b20_branch2a')\n .atrous_conv(3, 3, 256, 2, padding='SAME', biased=False, relu=False, name='res4b20_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b20_branch2b')\n .conv(1, 1, 1024, 1, 1, biased=False, relu=False, name='res4b20_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn4b20_branch2c'))\n\n (self.feed('res4b19_relu',\n 'bn4b20_branch2c')\n .add(name='res4b20')\n .relu(name='res4b20_relu')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res4b21_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b21_branch2a')\n .atrous_conv(3, 3, 256, 2, padding='SAME', biased=False, relu=False, name='res4b21_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b21_branch2b')\n .conv(1, 1, 1024, 1, 1, biased=False, relu=False, name='res4b21_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn4b21_branch2c'))\n\n (self.feed('res4b20_relu',\n 'bn4b21_branch2c')\n .add(name='res4b21')\n .relu(name='res4b21_relu')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='res4b22_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b22_branch2a')\n .atrous_conv(3, 3, 256, 4, padding='SAME', biased=False, relu=False, name='res4b22_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn4b22_branch2b')\n .conv(1, 1, 1024, 1, 1, biased=False, relu=False, name='res4b22_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn4b22_branch2c'))\n\n (self.feed('res4b21_relu',\n 'bn4b22_branch2c')\n .add(name='res4b22')\n .relu(name='res4b22_relu')\n .conv(1, 1, 2048, 1, 1, biased=False, relu=False, name='res5a_branch1')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn5a_branch1'))\n\n (self.feed('res4b22_relu')\n .conv(1, 1, 512, 1, 1, biased=False, relu=False, name='res5a_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn5a_branch2a')\n .atrous_conv(3, 3, 512, 4, padding='SAME', biased=False, relu=False, name='res5a_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn5a_branch2b')\n .conv(1, 1, 2048, 1, 1, biased=False, relu=False, name='res5a_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn5a_branch2c'))\n\n (self.feed('bn5a_branch1',\n 'bn5a_branch2c')\n .add(name='res5a')\n .relu(name='res5a_relu')\n .conv(1, 1, 512, 1, 1, biased=False, relu=False, name='res5b_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn5b_branch2a')\n .atrous_conv(3, 3, 512, 8, padding='SAME', biased=False, relu=False, name='res5b_branch2b')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn5b_branch2b')\n .conv(1, 1, 2048, 1, 1, biased=False, relu=False, name='res5b_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn5b_branch2c'))\n\n (self.feed('res5a_relu',\n 'bn5b_branch2c')\n .add(name='res5b')\n .relu(name='res5b_relu')\n .conv(1, 1, 512, 1, 1, biased=False, relu=False, name='res5c_branch2a')\n .batch_normalization(is_training=is_training, activation_fn=tf.nn.relu, name='bn5c_branch2a')\n .atrous_conv(3, 3, 512, 16, padding='SAME', biased=False, relu=False, name='res5c_branch2b')\n .batch_normalization(activation_fn=tf.nn.relu, name='bn5c_branch2b', is_training=is_training)\n .conv(1, 1, 2048, 1, 1, biased=False, relu=False, name='res5c_branch2c')\n .batch_normalization(is_training=is_training, activation_fn=None, name='bn5c_branch2c'))\n\n (self.feed('res5b_relu',\n 'bn5c_branch2c')\n .add(name='res5c')\n .relu(name='res5c_relu')\n .atrous_conv(3, 3, 256, 6, padding='SAME', relu=False, name='fc1_voc12_c0')\n .batch_normalization(is_training=is_training, activation_fn=None, name='fc1_voc12_c0_bn'))\n\n (self.feed('res5c_relu')\n .atrous_conv(3, 3, 256, 12, padding='SAME', relu=False, name='fc1_voc12_c1')\n .batch_normalization(is_training=is_training, activation_fn=None, name='fc1_voc12_c1_bn'))\n\n (self.feed('res5c_relu')\n .atrous_conv(3, 3, 256, 18, padding='SAME', relu=False, name='fc1_voc12_c2')\n .batch_normalization(is_training=is_training, activation_fn=None, name='fc1_voc12_c2_bn'))\n\n (self.feed('res5c_relu')\n .atrous_conv(1, 1, 256, 1, padding='SAME', relu=False, name='fc1_voc12_c3')\n .batch_normalization(is_training=is_training, activation_fn=None, name='fc1_voc12_c3_bn'))\n\n (self.feed('fc1_voc12_c0_bn',\n 'fc1_voc12_c1_bn',\n 'fc1_voc12_c2_bn',\n 'fc1_voc12_c3_bn')\n .add(name='fc1_voc12')\n .conv(1, 1, 256, 1, 1, biased=False, relu=False, name='fc_oooo')\n .batch_normalization(is_training=is_training, activation_fn=None, name='fc_oooo_bn')\n .conv(1, 1, num_classes, 1, 1, biased=False, relu=False, name='fc_out'))\n" }, { "path": "network.py", "content": "import numpy as np\nimport tensorflow as tf\nslim = tf.contrib.slim\n\nDEFAULT_PADDING = 'SAME'\n\n\ndef layer(op):\n '''Decorator for composable network layers.'''\n\n def layer_decorated(self, *args, **kwargs):\n # Automatically set a name if not provided.\n name = kwargs.setdefault('name', self.get_unique_name(op.__name__))\n # Figure out the layer inputs.\n if len(self.terminals) == 0:\n raise RuntimeError('No input variables found for layer %s.' % name)\n elif len(self.terminals) == 1:\n layer_input = self.terminals[0]\n else:\n layer_input = list(self.terminals)\n # Perform the operation and get the output.\n layer_output = op(self, layer_input, *args, **kwargs)\n # Add to layer LUT.\n self.layers[name] = layer_output\n # This output is now the input for the next layer.\n self.feed(layer_output)\n # Return self for chained calls.\n return self\n\n return layer_decorated\n\n\nclass Network(object):\n\n def __init__(self, inputs, trainable=True, is_training=False, num_classes=21):\n # The input nodes for this network\n self.inputs = inputs\n # The current list of terminal nodes\n self.terminals = []\n # Mapping from layer names to layers\n self.layers = dict(inputs)\n # If true, the resulting variables are set as trainable\n self.trainable = trainable\n # Switch variable for dropout\n self.use_dropout = tf.placeholder_with_default(tf.constant(1.0),\n shape=[],\n name='use_dropout')\n self.setup(is_training, num_classes)\n\n def setup(self, is_training):\n '''Construct the network. '''\n raise NotImplementedError('Must be implemented by the subclass.')\n\n def load(self, data_path, session, ignore_missing=False):\n '''Load network weights.\n data_path: The path to the numpy-serialized network weights\n session: The current TensorFlow session\n ignore_missing: If true, serialized weights for missing layers are ignored.\n '''\n data_dict = np.load(data_path).item()\n for op_name in data_dict:\n with tf.variable_scope(op_name, reuse=True):\n for param_name, data in data_dict[op_name].iteritems():\n try:\n var = tf.get_variable(param_name)\n session.run(var.assign(data))\n except ValueError:\n if not ignore_missing:\n raise\n\n def feed(self, *args):\n '''Set the input(s) for the next operation by replacing the terminal nodes.\n The arguments can be either layer names or the actual layers.\n '''\n assert len(args) != 0\n self.terminals = []\n for fed_layer in args:\n if isinstance(fed_layer, str):\n try:\n fed_layer = self.layers[fed_layer]\n except KeyError:\n raise KeyError('Unknown layer name fed: %s' % fed_layer)\n self.terminals.append(fed_layer)\n return self\n\n def get_output(self):\n '''Returns the current network output.'''\n return self.terminals[-1]\n\n def get_unique_name(self, prefix):\n '''Returns an index-suffixed unique name for the given prefix.\n This is used for auto-generating layer names based on the type-prefix.\n '''\n ident = sum(t.startswith(prefix) for t, _ in self.layers.items()) + 1\n return '%s_%d' % (prefix, ident)\n\n def make_var(self, name, shape):\n '''Creates a new TensorFlow variable.'''\n return tf.get_variable(name, shape, trainable=self.trainable)\n\n def validate_padding(self, padding):\n '''Verifies that the padding is one of the supported ones.'''\n assert padding in ('SAME', 'VALID')\n\n @layer\n def conv(self,\n input,\n k_h,\n k_w,\n c_o,\n s_h,\n s_w,\n name,\n relu=True,\n padding=DEFAULT_PADDING,\n group=1,\n biased=True):\n # Verify that the padding is acceptable\n self.validate_padding(padding)\n # Get the number of channels in the input\n c_i = input.get_shape()[-1]\n # Verify that the grouping parameter is valid\n assert c_i % group == 0\n assert c_o % group == 0\n # Convolution for a given input and kernel\n convolve = lambda i, k: tf.nn.conv2d(i, k, [1, s_h, s_w, 1], padding=padding)\n with tf.variable_scope(name) as scope:\n kernel = self.make_var('weights', shape=[k_h, k_w, c_i // group, c_o])\n if group == 1:\n # This is the common-case. Convolve the input without any further complications.\n output = convolve(input, kernel)\n else:\n # Split the input into groups and then convolve each of them independently\n input_groups = tf.split(3, group, input)\n kernel_groups = tf.split(3, group, kernel)\n output_groups = [convolve(i, k) for i, k in zip(input_groups, kernel_groups)]\n # Concatenate the groups\n output = tf.concat(3, output_groups)\n # Add the biases\n if biased:\n biases = self.make_var('biases', [c_o])\n output = tf.nn.bias_add(output, biases)\n if relu:\n # ReLU non-linearity\n output = tf.nn.relu(output, name=scope.name)\n return output\n\n @layer\n def atrous_conv(self,\n input,\n k_h,\n k_w,\n c_o,\n dilation,\n name,\n relu=True,\n padding=DEFAULT_PADDING,\n group=1,\n biased=True):\n # Verify that the padding is acceptable\n self.validate_padding(padding)\n # Get the number of channels in the input\n c_i = input.get_shape()[-1]\n # Verify that the grouping parameter is valid\n assert c_i % group == 0\n assert c_o % group == 0\n # Convolution for a given input and kernel\n convolve = lambda i, k: tf.nn.atrous_conv2d(i, k, dilation, padding=padding)\n with tf.variable_scope(name) as scope:\n kernel = self.make_var('weights', shape=[k_h, k_w, c_i // group, c_o])\n if group == 1:\n # This is the common-case. Convolve the input without any further complications.\n output = convolve(input, kernel)\n else:\n # Split the input into groups and then convolve each of them independently\n input_groups = tf.split(3, group, input)\n kernel_groups = tf.split(3, group, kernel)\n output_groups = [convolve(i, k) for i, k in zip(input_groups, kernel_groups)]\n # Concatenate the groups\n output = tf.concat(3, output_groups)\n # Add the biases\n if biased:\n biases = self.make_var('biases', [c_o])\n output = tf.nn.bias_add(output, biases)\n if relu:\n # ReLU non-linearity\n output = tf.nn.relu(output, name=scope.name)\n return output\n \n @layer\n def relu(self, input, name):\n return tf.nn.relu(input, name=name)\n\n @layer\n def max_pool(self, input, k_h, k_w, s_h, s_w, name, padding=DEFAULT_PADDING):\n self.validate_padding(padding)\n return tf.nn.max_pool(input,\n ksize=[1, k_h, k_w, 1],\n strides=[1, s_h, s_w, 1],\n padding=padding,\n name=name)\n\n @layer\n def avg_pool(self, input, k_h, k_w, s_h, s_w, name, padding=DEFAULT_PADDING):\n self.validate_padding(padding)\n return tf.nn.avg_pool(input,\n ksize=[1, k_h, k_w, 1],\n strides=[1, s_h, s_w, 1],\n padding=padding,\n name=name)\n\n @layer\n def lrn(self, input, radius, alpha, beta, name, bias=1.0):\n return tf.nn.local_response_normalization(input,\n depth_radius=radius,\n alpha=alpha,\n beta=beta,\n bias=bias,\n name=name)\n\n @layer\n def concat(self, inputs, axis, name):\n return tf.concat(axis=axis, values=inputs, name=name)\n\n @layer\n def add(self, inputs, name):\n return tf.add_n(inputs, name=name)\n\n @layer\n def fc(self, input, num_out, name, relu=True):\n with tf.variable_scope(name) as scope:\n input_shape = input.get_shape()\n if input_shape.ndims == 4:\n # The input is spatial. Vectorize it first.\n dim = 1\n for d in input_shape[1:].as_list():\n dim *= d\n feed_in = tf.reshape(input, [-1, dim])\n else:\n feed_in, dim = (input, input_shape[-1].value)\n weights = self.make_var('weights', shape=[dim, num_out])\n biases = self.make_var('biases', [num_out])\n op = tf.nn.relu_layer if relu else tf.nn.xw_plus_b\n fc = op(feed_in, weights, biases, name=scope.name)\n return fc\n\n @layer\n def softmax(self, input, name):\n input_shape = map(lambda v: v.value, input.get_shape())\n if len(input_shape) > 2:\n # For certain models (like NiN), the singleton spatial dimensions\n # need to be explicitly squeezed, since they're not broadcast-able\n # in TensorFlow's NHWC ordering (unlike Caffe's NCHW).\n if input_shape[1] == 1 and input_shape[2] == 1:\n input = tf.squeeze(input, squeeze_dims=[1, 2])\n else:\n raise ValueError('Rank 2 tensor input expected for softmax!')\n return tf.nn.softmax(input, name)\n \n @layer\n def batch_normalization(self, input, name, is_training, activation_fn=None, scale=True):\n with tf.variable_scope(name) as scope:\n output = slim.batch_norm(\n input,\n activation_fn=activation_fn,\n is_training=is_training,\n updates_collections=None,\n scale=scale,\n scope=scope)\n return output\n\n @layer\n def dropout(self, input, keep_prob, name):\n keep = 1 - self.use_dropout + (self.use_dropout * keep_prob)\n return tf.nn.dropout(input, keep, name=name)\n\n @layer\n def resize_bilinear(self, input, size, name):\n return tf.image.resize_bilinear(input, size=size, name=name)\n\n @layer\n def global_average_pooling(self, input, name):\n \"\"\"\n shape = input.get_shape().as_list()\n _mean = tf.reduce_mean(input, [1, 2], name='GAP', keep_dims=True)\n _conv = self.conv(_mean, 1, 1, 256, 1, 1, biased=False, relu=False, name='GAP_conv')\n _bn = self.batch_normalization(_conv, is_training=is_training, activation_fn=None, name='GAP_conv_bn')\n _resize = tf.image.resize_bilinear(_bn, size=shape, name=name)\n \"\"\"\n\n return tf.reduce_mean(input, [1, 2], name=name, keep_dims=True)\n" }, { "path": "requirements.txt", "content": "tensorflow-gpu==1.15.0\nscipy==1.7.3\npillow==9.1.0\ngdown\n" } ]