Repository: BitconFeng/Deep-Feature-video Branch: master Commit: fff73fbcd0e2 Files: 125 Total size: 1.0 MB Directory structure: gitextract_zxbt3wtx/ ├── LICENSE ├── README.md ├── ThirdPartyNotices.txt ├── dff_rfcn/ │ ├── __init__.py │ ├── _init_paths.py │ ├── config/ │ │ ├── __init__.py │ │ └── config.py │ ├── core/ │ │ ├── DataParallelExecutorGroup.py │ │ ├── __init__.py │ │ ├── callback.py │ │ ├── loader.py │ │ ├── metric.py │ │ ├── module.py │ │ ├── rcnn.py │ │ └── tester.py │ ├── demo.py │ ├── demo_batch.py │ ├── function/ │ │ ├── __init__.py │ │ ├── test_rcnn.py │ │ ├── test_rpn.py │ │ ├── train_rcnn.py │ │ └── train_rpn.py │ ├── operator_cxx/ │ │ ├── multi_proposal-inl.h │ │ ├── multi_proposal.cc │ │ ├── multi_proposal.cu │ │ ├── psroi_pooling-inl.h │ │ ├── psroi_pooling.cc │ │ └── psroi_pooling.cu │ ├── operator_py/ │ │ ├── __init__.py │ │ ├── box_annotator_ohem.py │ │ ├── proposal.py │ │ ├── proposal_target.py │ │ ├── rpn_inv_normalize.py │ │ └── tile_as.py │ ├── symbols/ │ │ ├── __init__.py │ │ └── resnet_v1_101_flownet_rfcn.py │ ├── test.py │ └── train_end2end.py ├── experiments/ │ ├── dff_rfcn/ │ │ ├── cfgs/ │ │ │ ├── dff_rfcn_vid_demo.yaml │ │ │ └── resnet_v1_101_flownet_imagenet_vid_rfcn_end2end_ohem.yaml │ │ ├── dff_rfcn_end2end_train_test.py │ │ └── dff_rfcn_test.py │ └── rfcn/ │ ├── cfgs/ │ │ ├── resnet_v1_101_imagenet_vid_rfcn_end2end_ohem.yaml │ │ └── rfcn_vid_demo.yaml │ ├── rfcn_end2end_train_test.py │ └── rfcn_test.py ├── green2.py ├── init.bat ├── init.sh ├── lib/ │ ├── Makefile │ ├── __init__.py │ ├── bbox/ │ │ ├── .gitignore │ │ ├── __init__.py │ │ ├── bbox.pyx │ │ ├── bbox_regression.py │ │ ├── bbox_transform.py │ │ ├── setup_linux.py │ │ └── setup_windows.py │ ├── dataset/ │ │ ├── __init__.py │ │ ├── ds_utils.py │ │ ├── imagenet_vid.py │ │ ├── imagenet_vid_eval.py │ │ └── imdb.py │ ├── nms/ │ │ ├── __init__.py │ │ ├── cpu_nms.pyx │ │ ├── gpu_nms.cu │ │ ├── gpu_nms.hpp │ │ ├── gpu_nms.pyx │ │ ├── nms.py │ │ ├── nms_kernel.cu │ │ ├── setup_linux.py │ │ ├── setup_windows.py │ │ └── setup_windows_cuda.py │ ├── rpn/ │ │ ├── __init__.py │ │ ├── generate_anchor.py │ │ └── rpn.py │ └── utils/ │ ├── PrefetchingIter.py │ ├── __init__.py │ ├── combine_model.py │ ├── create_logger.py │ ├── image.py │ ├── image_processing.py │ ├── load_data.py │ ├── load_model.py │ ├── lr_scheduler.py │ ├── roidb.py │ ├── save_model.py │ ├── show_boxes.py │ ├── symbol.py │ └── tictoc.py ├── rfcn/ │ ├── __init__.py │ ├── _init_paths.py │ ├── config/ │ │ ├── __init__.py │ │ └── config.py │ ├── core/ │ │ ├── DataParallelExecutorGroup.py │ │ ├── __init__.py │ │ ├── callback.py │ │ ├── loader.py │ │ ├── metric.py │ │ ├── module.py │ │ ├── rcnn.py │ │ └── tester.py │ ├── demo.py │ ├── demo_batch.py │ ├── function/ │ │ ├── __init__.py │ │ ├── test_rcnn.py │ │ ├── test_rpn.py │ │ ├── train_rcnn.py │ │ └── train_rpn.py │ ├── operator_cxx/ │ │ ├── multi_proposal-inl.h │ │ ├── multi_proposal.cc │ │ ├── multi_proposal.cu │ │ ├── psroi_pooling-inl.h │ │ ├── psroi_pooling.cc │ │ └── psroi_pooling.cu │ ├── operator_py/ │ │ ├── __init__.py │ │ ├── box_annotator_ohem.py │ │ ├── proposal.py │ │ ├── proposal_target.py │ │ └── rpn_inv_normalize.py │ ├── symbols/ │ │ ├── __init__.py │ │ └── resnet_v1_101_rfcn.py │ ├── test.py │ └── train_end2end.py └── zero.md ================================================ FILE CONTENTS ================================================ ================================================ FILE: LICENSE ================================================ The MIT License (MIT) Copyright (c) 2017 Microsoft Corporation Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================ # Deep Feature Flow for Video Recognition ## Introduction **Deep Feature Flow** is initially described in a [CVPR 2017 paper] It provides a simple, fast, accurate, and end-to-end framework for video recognition (e.g., object detection and semantic segmentation in videos). It is worth noting that: * Deep Feature Flow significantly speeds up video recognition by applying the heavy-weight image recognition network (e.g., ResNet-101) on sparse key frames, and propagating the recognition outputs (feature maps) to the other frames by the light-weight flow network (e.g., [FlowNet]. * The entire system is end-to-end trained for the task of video recognition, which is vital for improving the recognition accuracy. Directly adopting state-of-the-art flow estimation methods without end-to-end training would deliver noticable worse results. * Deep Feature Flow can easily make use of sparsely annotated video recognition datasets, where only a small portion of the frames are annotated with ground-truth labels. ***Click image to watch our demo video*** [![Demo Video on YouTube](https://media.giphy.com/media/14erFWP6f5tDVe/giphy.gif)] [![Demo Video on YouTube](https://media.giphy.com/media/xwB5LVfIjLtS/giphy.gif)] ## Disclaimer This is an official implementation for [Deep Feature Flow for Video Recognition](https://arxiv.org/abs/1611.07715) (DFF) based on MXNet. It is worth noticing that: * The original implementation is based on our internal Caffe version on Windows. There are slight differences in the final accuracy and running time due to the plenty details in platform switch. ## License © Microsoft, 2018. Licensed under the [MIT](LICENSE) License. ## Citing Deep Feature Flow If you find Deep Feature Flow useful in your research, please consider citing: ``` @inproceedings{zhu17dff, Author = {Xizhou Zhu, Yuwen Xiong, Jifeng Dai, Lu Yuan, Yichen Wei}, Title = {Deep Feature Flow for Video Recognition}, Conference = {CVPR}, Year = {2017} } @inproceedings{dai16rfcn, Author = {Jifeng Dai, Yi Li, Kaiming He, Jian Sun}, Title = {{R-FCN}: Object Detection via Region-based Fully Convolutional Networks}, Conference = {NIPS}, Year = {2016} } ``` ## Main Results | | training data | testing data | mAP@0.5 | time/image
(Tesla K40) | time/image
(Maxwell Titan X) | |---------------------------------|-------------------|--------------|---------|---------|--------| | Frame baseline
(R-FCN, ResNet-v1-101) | ImageNet DET train + VID train | ImageNet VID validation | 74.1 | 0.271s | 0.133s | | Deep Feature Flow
(R-FCN, ResNet-v1-101, FlowNet) | ImageNet DET train + VID train | ImageNet VID validation | 73.0 | 0.073s | 0.034s | *Running time is counted on a single GPU (mini-batch size is 1 in inference, key-frame duration length for Deep Feature Flow is 10).* *The runtime of the light-weight FlowNet seems to be a bit slower on MXNet than that on Caffe.* ## Requirements: Software 1. MXNet from Due to the rapid development of MXNet, it is recommended to checkout this version if you encounter any issues. We may maintain this repository periodically if MXNet adds important feature in future release. 2. Python 2.7. We recommend using Anaconda2 as it already includes many common packages. We do not suppoort Python 3 yet, if you want to use Python 3 you need to modify the code to make it work. 3. Python packages might missing: cython, opencv-python >= 3.2.0, easydict. If `pip` is set up on your system, those packages should be able to be fetched and installed by running ``` pip install Cython pip install opencv-python==3.2.0.6 pip install easydict==1.6 ``` 4. For Windows users, Visual Studio 2015 is needed to compile cython module. ## Requirements: Hardware Any NVIDIA GPUs with at least 6GB memory should be OK ## Installation 1. Clone the Deep Feature Flow repository, and we'll call the directory that you cloned Deep-Feature-Flow as ${DFF_ROOT}. ~~~ git clone https://github.com/msracver/Deep-Feature-Flow.git ~~~ 2. For Windows users, run ``cmd .\init.bat``. For Linux user, run `sh ./init.sh`. The scripts will build cython module automatically and create some folders. 3. Install MXNet: 3.1 Clone MXNet and checkout to [MXNet@(commit 62ecb60)] by ``` git clone --recursive https://github.com/dmlc/mxnet.git git checkout 62ecb60 git submodule update ``` 3.2 Copy operators in `$(DFF_ROOT)/dff_rfcn/operator_cxx` or `$(DFF_ROOT)/rfcn/operator_cxx` to `$(YOUR_MXNET_FOLDER)/src/operator/contrib` by ``` cp -r $(DFF_ROOT)/dff_rfcn/operator_cxx/* $(MXNET_ROOT)/src/operator/contrib/ ``` 3.3 Compile MXNet ``` cd ${MXNET_ROOT} make -j4 ``` 3.4 Install the MXNet Python binding by ***Note: If you will actively switch between different versions of MXNet, please follow 3.5 instead of 3.4*** ``` cd python sudo python setup.py install ``` 3.5 For advanced users, you may put your Python packge into `./external/mxnet/$(YOUR_MXNET_PACKAGE)`, and modify `MXNET_VERSION` in `./experiments/dff_rfcn/cfgs/*.yaml` to `$(YOUR_MXNET_PACKAGE)`. Thus you can switch among different versions of MXNet quickly. ## Demo 1. To run the demo with our trained model (on ImageNet DET + VID train), please download the model manually from [OneDrive](for users from Mainland China, please try [Baidu Yun]), and put it under folder `model/`. Make sure it looks like this: ``` ./model/rfcn_vid-0000.params ./model/rfcn_dff_flownet_vid-0000.params ``` 2. Run (inference batch size = 1) ``` python ./rfcn/demo.py python ./dff_rfcn/demo.py ``` or run (inference batch size = 10) ``` python ./rfcn/demo_batch.py python ./dff_rfcn/demo_batch.py ``` ## Preparation for Training & Testing 1. Please download ILSVRC2015 DET and ILSVRC2015 VID dataset, and make sure it looks like this: ``` ./data/ILSVRC2015/ ./data/ILSVRC2015/Annotations/DET ./data/ILSVRC2015/Annotations/VID ./data/ILSVRC2015/Data/DET ./data/ILSVRC2015/Data/VID ./data/ILSVRC2015/ImageSets ``` ## FAQ Q: It says `AttributeError: 'module' object has no attribute 'MultiProposal'`. A: This is because either - you forget to copy the operators to your MXNet folder - or you copy to the wrong path - or you forget to re-compile and install - or you install the wrong MXNet Please print `mxnet.__path__` to make sure you use correct MXNet

Q: I encounter `segment fault` at the beginning. A: A compatibility issue has been identified between MXNet and opencv-python 3.0+. We suggest that you always `import cv2` first before `import mxnet` in the entry script.

Q: I find the training speed becomes slower when training for a long time. A: It has been identified that MXNet on Windows has this problem. So we recommend to run this program on Linux. You could also stop it and resume the training process to regain the training speed if you encounter this problem.

Q: Can you share your caffe implementation? A: Due to several reasons (code is based on a old, internal Caffe, port to public Caffe needs extra work, time limit, etc.). We do not plan to release our Caffe code. Since a warping layer is easy to implement, anyone who wish to do it is welcome to make a pull request. ================================================ FILE: ThirdPartyNotices.txt ================================================ Deep Feature Flow THIRD-PARTY SOFTWARE NOTICES AND INFORMATION This project incorporates components from the projects listed below. The original copyright notices and the licenses under which Microsoft received such components are set forth below. Microsoft reserves all rights not expressly granted herein, whether by implication, estoppel or otherwise. 1. MXNet (https://github.com/apache/incubator-mxnet) 2. Fast R-CNN (https://github.com/rbgirshick/fast-rcnn) 3. Faster R-CNN (https://github.com/rbgirshick/py-faster-rcnn) 4. MS COCO API (https://github.com/cocodataset/cocoapi) MXNet Copyright (c) 2015-2016 by Contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Fast R-CNN Copyright (c) Microsoft Corporation All rights reserved. MIT License Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Faster R-CNN The MIT License (MIT) Copyright (c) 2015 Microsoft Corporation Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. MS COCO API Copyright (c) 2014, Piotr Dollar and Tsung-Yi Lin All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. The views and conclusions contained in the software and documentation are those of the authors and should not be interpreted as representing official policies, either expressed or implied, of the FreeBSD Project. ================================================ FILE: dff_rfcn/__init__.py ================================================ ================================================ FILE: dff_rfcn/_init_paths.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong # -------------------------------------------------------- import os.path as osp import sys def add_path(path): if path not in sys.path: sys.path.insert(0, path) this_dir = osp.dirname(__file__) lib_path = osp.join(this_dir, '..', 'lib') add_path(lib_path) ================================================ FILE: dff_rfcn/config/__init__.py ================================================ ================================================ FILE: dff_rfcn/config/config.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Xizhou Zhu, Yuwen Xiong, Bin Xiao # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- import yaml import numpy as np from easydict import EasyDict as edict config = edict() config.MXNET_VERSION = '' config.output_path = '' config.symbol = '' config.gpus = '' config.CLASS_AGNOSTIC = True config.SCALES = [(600, 1000)] # first is scale (the shorter side); second is max size # default training config.default = edict() config.default.frequent = 20 config.default.kvstore = 'device' # network related params config.network = edict() config.network.pretrained = '' config.network.pretrained_flow = '' config.network.pretrained_epoch = 0 config.network.PIXEL_MEANS = np.array([0, 0, 0]) config.network.IMAGE_STRIDE = 0 config.network.RPN_FEAT_STRIDE = 16 config.network.RCNN_FEAT_STRIDE = 16 config.network.FIXED_PARAMS = ['gamma', 'beta'] config.network.ANCHOR_SCALES = (8, 16, 32) config.network.ANCHOR_RATIOS = (0.5, 1, 2) config.network.NORMALIZE_RPN = True config.network.ANCHOR_MEANS = (0.0, 0.0, 0.0, 0.0) config.network.ANCHOR_STDS = (0.1, 0.1, 0.4, 0.4) config.network.NUM_ANCHORS = len(config.network.ANCHOR_SCALES) * len(config.network.ANCHOR_RATIOS) config.network.DFF_FEAT_DIM = 1024 # dataset related params config.dataset = edict() config.dataset.dataset = 'ImageNetVID' config.dataset.image_set = 'DET_train_30classes+VID_train_15frames' config.dataset.test_image_set = 'VID_val_videos' config.dataset.root_path = './data' config.dataset.dataset_path = './data/ILSVRC2015' config.dataset.NUM_CLASSES = 31 config.TRAIN = edict() config.TRAIN.lr = 0 config.TRAIN.lr_step = '' config.TRAIN.lr_factor = 0.1 config.TRAIN.warmup = False config.TRAIN.warmup_lr = 0 config.TRAIN.warmup_step = 0 config.TRAIN.momentum = 0.9 config.TRAIN.wd = 0.0005 config.TRAIN.begin_epoch = 0 config.TRAIN.end_epoch = 0 config.TRAIN.model_prefix = '' # whether resume training config.TRAIN.RESUME = False # whether flip image config.TRAIN.FLIP = True # whether shuffle image config.TRAIN.SHUFFLE = True # whether use OHEM config.TRAIN.ENABLE_OHEM = False # size of images for each device, 2 for rcnn, 1 for rpn and e2e config.TRAIN.BATCH_IMAGES = 2 # e2e changes behavior of anchor loader and metric config.TRAIN.END2END = False # group images with similar aspect ratio config.TRAIN.ASPECT_GROUPING = True # R-CNN # rcnn rois batch size config.TRAIN.BATCH_ROIS = 128 config.TRAIN.BATCH_ROIS_OHEM = 128 # rcnn rois sampling params config.TRAIN.FG_FRACTION = 0.25 config.TRAIN.FG_THRESH = 0.5 config.TRAIN.BG_THRESH_HI = 0.5 config.TRAIN.BG_THRESH_LO = 0.0 # rcnn bounding box regression params config.TRAIN.BBOX_REGRESSION_THRESH = 0.5 config.TRAIN.BBOX_WEIGHTS = np.array([1.0, 1.0, 1.0, 1.0]) # RPN anchor loader # rpn anchors batch size config.TRAIN.RPN_BATCH_SIZE = 256 # rpn anchors sampling params config.TRAIN.RPN_FG_FRACTION = 0.5 config.TRAIN.RPN_POSITIVE_OVERLAP = 0.7 config.TRAIN.RPN_NEGATIVE_OVERLAP = 0.3 config.TRAIN.RPN_CLOBBER_POSITIVES = False # rpn bounding box regression params config.TRAIN.RPN_BBOX_WEIGHTS = (1.0, 1.0, 1.0, 1.0) config.TRAIN.RPN_POSITIVE_WEIGHT = -1.0 # used for end2end training # RPN proposal config.TRAIN.CXX_PROPOSAL = True config.TRAIN.RPN_NMS_THRESH = 0.7 config.TRAIN.RPN_PRE_NMS_TOP_N = 12000 config.TRAIN.RPN_POST_NMS_TOP_N = 2000 config.TRAIN.RPN_MIN_SIZE = config.network.RPN_FEAT_STRIDE # approximate bounding box regression config.TRAIN.BBOX_NORMALIZATION_PRECOMPUTED = True config.TRAIN.BBOX_MEANS = (0.0, 0.0, 0.0, 0.0) config.TRAIN.BBOX_STDS = (0.1, 0.1, 0.2, 0.2) # DFF, trained image sampled from [min_offset, max_offset] config.TRAIN.MIN_OFFSET = -9 config.TRAIN.MAX_OFFSET = 0 config.TEST = edict() # R-CNN testing # use rpn to generate proposal config.TEST.HAS_RPN = False # size of images for each device config.TEST.BATCH_IMAGES = 1 # RPN proposal config.TEST.CXX_PROPOSAL = True config.TEST.RPN_NMS_THRESH = 0.7 config.TEST.RPN_PRE_NMS_TOP_N = 6000 config.TEST.RPN_POST_NMS_TOP_N = 300 config.TEST.RPN_MIN_SIZE = config.network.RPN_FEAT_STRIDE # RCNN nms config.TEST.NMS = 0.3 # DFF config.TEST.KEY_FRAME_INTERVAL = 10 config.TEST.max_per_image = 300 # Test Model Epoch config.TEST.test_epoch = 0 def update_config(config_file): exp_config = None with open(config_file) as f: exp_config = edict(yaml.load(f)) for k, v in exp_config.items(): if k in config: if isinstance(v, dict): if k == 'TRAIN': if 'BBOX_WEIGHTS' in v: v['BBOX_WEIGHTS'] = np.array(v['BBOX_WEIGHTS']) elif k == 'network': if 'PIXEL_MEANS' in v: v['PIXEL_MEANS'] = np.array(v['PIXEL_MEANS']) for vk, vv in v.items(): config[k][vk] = vv else: if k == 'SCALES': config[k][0] = (tuple(v)) else: config[k] = v else: raise ValueError("key must exist in config.py") ================================================ FILE: dff_rfcn/core/DataParallelExecutorGroup.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- import logging import numpy as np from mxnet import context as ctx from mxnet import ndarray as nd from mxnet.io import DataDesc from mxnet.executor_manager import _split_input_slice def _load_general(data, targets, major_axis): """Load a list of arrays into a list of arrays specified by slices""" for d_src, d_targets in zip(data, targets): if isinstance(d_targets, nd.NDArray): d_src.copyto(d_targets) elif isinstance(d_src, (list, tuple)): for src, dst in zip(d_src, d_targets): src.copyto(dst) else: raise NotImplementedError def _load_data(batch, targets, major_axis): """Load data into sliced arrays""" _load_general(batch.data, targets, major_axis) def _load_label(batch, targets, major_axis): """Load label into sliced arrays""" _load_general(batch.label, targets, major_axis) def _merge_multi_context(outputs, major_axis): """Merge outputs that lives on multiple context into one, so that they look like living on one context. """ rets = [] for tensors, axis in zip(outputs, major_axis): if axis >= 0: rets.append(nd.concatenate(tensors, axis=axis, always_copy=False)) else: # negative axis means the there is no batch_size axis, and all the # results should be the same on each device. We simply take the # first one, without checking they are actually the same rets.append(tensors[0]) return rets class DataParallelExecutorGroup(object): """DataParallelExecutorGroup is a group of executors that lives on a group of devices. This is a helper class used to implement data parallelization. Each mini-batch will be split and run on the devices. Parameters ---------- symbol : Symbol The common symbolic computation graph for all executors. contexts : list A list of contexts. workload : list If not `None`, could be a list of numbers that specify the workload to be assigned to different context. Larger number indicate heavier workload. data_shapes : list Should be a list of (name, shape) tuples, for the shapes of data. Note the order is important and should be the same as the order that the `DataIter` provide the data. label_shapes : list Should be a list of (name, shape) tuples, for the shapes of label. Note the order is important and should be the same as the order that the `DataIter` provide the label. param_names : list A list of strings, indicating the names of parameters (e.g. weights, filters, etc.) in the computation graph. for_training : bool Indicate whether the executors should be bind for training. When not doing training, the memory for gradients will not be allocated. inputs_need_grad : bool Indicate whether the gradients for the input data should be computed. This is currently not used. It will be useful for implementing composition of modules. shared_group : DataParallelExecutorGroup Default is `None`. This is used in bucketing. When not `None`, it should be a executor group corresponding to a different bucket. In other words, it will correspond to a different symbol but with the same set of parameters (e.g. unrolled RNNs with different lengths). In this case, many memory will be shared. logger : Logger Default is `logging`. fixed_param_names: list of str Indicate parameters to be fixed during training. Parameters in this list will not allocate space for gradient, nor do gradient calculation. grad_req : str, list of str, dict of str to str Requirement for gradient accumulation. Can be 'write', 'add', or 'null' (default to 'write'). Can be specified globally (str) or for each argument (list, dict). """ def __init__(self, symbol, contexts, workload, data_shapes, label_shapes, param_names, for_training, inputs_need_grad, shared_group=None, logger=logging, fixed_param_names=None, grad_req='write', state_names=None): self.param_names = param_names self.arg_names = symbol.list_arguments() self.aux_names = symbol.list_auxiliary_states() self.symbol = symbol self.contexts = contexts self.workload = workload self.for_training = for_training self.inputs_need_grad = inputs_need_grad self.logger = logger #In the future we should have a better way to profile memory per device (haibin) # self._total_exec_bytes = 0 self.fixed_param_names = fixed_param_names if self.fixed_param_names is None: self.fixed_param_names = [] self.state_names = state_names if self.state_names is None: self.state_names = [] if not for_training: grad_req = 'null' # data_shapes = [x if isinstance(x, DataDesc) else DataDesc(*x) for x in data_shapes] # if label_shapes is not None: # label_shapes = [x if isinstance(x, DataDesc) else DataDesc(*x) for x in label_shapes] data_names = [x.name for x in data_shapes[0]] if isinstance(grad_req, str): self.grad_req = {} for k in self.arg_names: if k in self.param_names: self.grad_req[k] = 'null' if k in self.fixed_param_names else grad_req elif k in data_names: self.grad_req[k] = grad_req if self.inputs_need_grad else 'null' else: self.grad_req[k] = 'null' elif isinstance(grad_req, (list, tuple)): assert len(grad_req) == len(self.arg_names) self.grad_req = dict(zip(self.arg_names, grad_req)) elif isinstance(grad_req, dict): self.grad_req = {} for k in self.arg_names: if k in self.param_names: self.grad_req[k] = 'null' if k in self.fixed_param_names else 'write' elif k in data_names: self.grad_req[k] = 'write' if self.inputs_need_grad else 'null' else: self.grad_req[k] = 'null' self.grad_req.update(grad_req) else: raise ValueError("grad_req must be one of str, list, tuple, or dict.") if shared_group is not None: self.shared_data_arrays = shared_group.shared_data_arrays else: self.shared_data_arrays = [{} for _ in contexts] # initialize some instance variables self.batch_size = len(data_shapes) self.slices = None self.execs = [] self._default_execs = None self.data_arrays = None self.label_arrays = None self.param_arrays = None self.state_arrays = None self.grad_arrays = None self.aux_arrays = None self.input_grad_arrays = None self.data_shapes = None self.label_shapes = None self.data_layouts = None self.label_layouts = None self.output_layouts = [DataDesc.get_batch_axis(self.symbol[name].attr('__layout__')) for name in self.symbol.list_outputs()] self.bind_exec(data_shapes, label_shapes, shared_group) def decide_slices(self, data_shapes): """Decide the slices for each context according to the workload. Parameters ---------- data_shapes : list list of (name, shape) specifying the shapes for the input data or label. """ assert len(data_shapes) > 0 major_axis = [DataDesc.get_batch_axis(x.layout) for x in data_shapes] for (name, shape), axis in zip(data_shapes, major_axis): if axis == -1: continue batch_size = shape[axis] if self.batch_size is not None: assert batch_size == self.batch_size, ("all data must have the same batch size: " + ("batch_size = %d, but " % self.batch_size) + ("%s has shape %s" % (name, shape))) else: self.batch_size = batch_size self.slices = _split_input_slice(self.batch_size, self.workload) return major_axis def _collect_arrays(self): """Collect internal arrays from executors.""" # convenient data structures self.data_arrays = [[e.arg_dict[name] for name, _ in self.data_shapes[0]] for e in self.execs] self.state_arrays = [[e.arg_dict[name] for e in self.execs] for name in self.state_names] if self.label_shapes is not None: self.label_arrays = [[e.arg_dict[name] for name, _ in self.label_shapes[0]] for e in self.execs] else: self.label_arrays = None self.param_arrays = [[exec_.arg_arrays[i] for exec_ in self.execs] for i, name in enumerate(self.arg_names) if name in self.param_names] if self.for_training: self.grad_arrays = [[exec_.grad_arrays[i] for exec_ in self.execs] for i, name in enumerate(self.arg_names) if name in self.param_names] else: self.grad_arrays = None data_names = [x[0] for x in self.data_shapes] if self.inputs_need_grad: self.input_grad_arrays = [[exec_.grad_arrays[i] for exec_ in self.execs] for i, name in enumerate(self.arg_names) if name in data_names] else: self.input_grad_arrays = None self.aux_arrays = [[exec_.aux_arrays[i] for exec_ in self.execs] for i in range(len(self.aux_names))] def bind_exec(self, data_shapes, label_shapes, shared_group=None, reshape=False): """Bind executors on their respective devices. Parameters ---------- data_shapes : list label_shapes : list shared_group : DataParallelExecutorGroup reshape : bool """ assert reshape or not self.execs for i in range(len(self.contexts)): data_shapes_i = data_shapes[i] if label_shapes is not None: label_shapes_i = label_shapes[i] else: label_shapes_i = [] if reshape: self.execs[i] = self._default_execs[i].reshape( allow_up_sizing=True, **dict(data_shapes_i + label_shapes_i)) else: self.execs.append(self._bind_ith_exec(i, data_shapes_i, label_shapes_i, shared_group)) self.data_shapes = data_shapes self.label_shapes = label_shapes self._collect_arrays() def reshape(self, data_shapes, label_shapes): """Reshape executors. Parameters ---------- data_shapes : list label_shapes : list """ if self._default_execs is None: self._default_execs = [i for i in self.execs] for i in range(len(self.contexts)): self.execs[i] = self._default_execs[i].reshape( allow_up_sizing=True, **dict(data_shapes[i] + (label_shapes[i] if label_shapes is not None else [])) ) self.data_shapes = data_shapes self.label_shapes = label_shapes self._collect_arrays() def set_params(self, arg_params, aux_params): """Assign, i.e. copy parameters to all the executors. Parameters ---------- arg_params : dict A dictionary of name to `NDArray` parameter mapping. aux_params : dict A dictionary of name to `NDArray` auxiliary variable mapping. """ for exec_ in self.execs: exec_.copy_params_from(arg_params, aux_params) def get_params(self, arg_params, aux_params): """ Copy data from each executor to `arg_params` and `aux_params`. Parameters ---------- arg_params : list of NDArray target parameter arrays aux_params : list of NDArray target aux arrays Notes ----- - This function will inplace update the NDArrays in arg_params and aux_params. """ for name, block in zip(self.param_names, self.param_arrays): weight = sum(w.copyto(ctx.cpu()) for w in block) / len(block) weight.astype(arg_params[name].dtype).copyto(arg_params[name]) for name, block in zip(self.aux_names, self.aux_arrays): weight = sum(w.copyto(ctx.cpu()) for w in block) / len(block) weight.astype(aux_params[name].dtype).copyto(aux_params[name]) def forward(self, data_batch, is_train=None): """Split `data_batch` according to workload and run forward on each devices. Parameters ---------- data_batch : DataBatch Or could be any object implementing similar interface. is_train : bool The hint for the backend, indicating whether we are during training phase. Default is `None`, then the value `self.for_training` will be used. Returns ------- """ _load_data(data_batch, self.data_arrays, self.data_layouts) if is_train is None: is_train = self.for_training if self.label_arrays is not None: assert not is_train or data_batch.label if data_batch.label: _load_label(data_batch, self.label_arrays, self.label_layouts) for exec_ in self.execs: exec_.forward(is_train=is_train) def get_outputs(self, merge_multi_context=True): """Get outputs of the previous forward computation. Parameters ---------- merge_multi_context : bool Default is `True`. In the case when data-parallelism is used, the outputs will be collected from multiple devices. A `True` value indicate that we should merge the collected results so that they look like from a single executor. Returns ------- If `merge_multi_context` is `True`, it is like `[out1, out2]`. Otherwise, it is like `[[out1_dev1, out1_dev2], [out2_dev1, out2_dev2]]`. All the output elements are `NDArray`. """ outputs = [[exec_.outputs[i] for exec_ in self.execs] for i in range(len(self.execs[0].outputs))] if merge_multi_context: outputs = _merge_multi_context(outputs, self.output_layouts) return outputs def get_states(self, merge_multi_context=True): """Get states from all devices Parameters ---------- merge_multi_context : bool Default is `True`. In the case when data-parallelism is used, the states will be collected from multiple devices. A `True` value indicate that we should merge the collected results so that they look like from a single executor. Returns ------- If `merge_multi_context` is `True`, it is like `[out1, out2]`. Otherwise, it is like `[[out1_dev1, out1_dev2], [out2_dev1, out2_dev2]]`. All the output elements are `NDArray`. """ assert not merge_multi_context, \ "merge_multi_context=True is not supported for get_states yet." return self.state_arrays def set_states(self, states=None, value=None): """Set value for states. Only one of states & value can be specified. Parameters ---------- states : list of list of NDArrays source states arrays formatted like [[state1_dev1, state1_dev2], [state2_dev1, state2_dev2]]. value : number a single scalar value for all state arrays. """ if states is not None: assert value is None, "Only one of states & value can be specified." _load_general(states, self.state_arrays, (0,)*len(states)) else: assert value is not None, "At least one of states & value must be specified." assert states is None, "Only one of states & value can be specified." for d_dst in self.state_arrays: for dst in d_dst: dst[:] = value def get_input_grads(self, merge_multi_context=True): """Get the gradients with respect to the inputs of the module. Parameters ---------- merge_multi_context : bool Default is `True`. In the case when data-parallelism is used, the outputs will be collected from multiple devices. A `True` value indicate that we should merge the collected results so that they look like from a single executor. Returns ------- If `merge_multi_context` is `True`, it is like `[grad1, grad2]`. Otherwise, it is like `[[grad1_dev1, grad1_dev2], [grad2_dev1, grad2_dev2]]`. All the output elements are `NDArray`. """ assert self.inputs_need_grad if merge_multi_context: return _merge_multi_context(self.input_grad_arrays, self.data_layouts) return self.input_grad_arrays def backward(self, out_grads=None): """Run backward on all devices. A backward should be called after a call to the forward function. Backward cannot be called unless `self.for_training` is `True`. Parameters ---------- out_grads : NDArray or list of NDArray, optional Gradient on the outputs to be propagated back. This parameter is only needed when bind is called on outputs that are not a loss function. """ assert self.for_training, 're-bind with for_training=True to run backward' if out_grads is None: out_grads = [] for i, exec_ in enumerate(self.execs): out_grads_slice = [] exec_.backward(out_grads=out_grads_slice) def update_metric(self, eval_metric, labels): """Accumulate the performance according to `eval_metric` on all devices. Parameters ---------- eval_metric : EvalMetric The metric used for evaluation. labels : list of NDArray Typically comes from `label` of a `DataBatch`. """ for texec, labels in zip(self.execs, labels): eval_metric.update(labels, texec.outputs) def _bind_ith_exec(self, i, data_shapes, label_shapes, shared_group): """Internal utility function to bind the i-th executor. """ shared_exec = None if shared_group is None else shared_group.execs[i] context = self.contexts[i] shared_data_arrays = self.shared_data_arrays[i] input_shapes = dict(data_shapes) if label_shapes is not None: input_shapes.update(dict(label_shapes)) arg_shapes, _, aux_shapes = self.symbol.infer_shape(**input_shapes) assert arg_shapes is not None, "shape inference failed" input_types = {x.name: x.dtype for x in data_shapes} if label_shapes is not None: input_types.update({x.name: x.dtype for x in label_shapes}) arg_types, _, aux_types = self.symbol.infer_type(**input_types) assert arg_types is not None, "type inference failed" arg_arrays = [] grad_arrays = {} if self.for_training else None def _get_or_reshape(name, shared_data_arrays, arg_shape, arg_type, context, logger): """Internal helper to get a memory block or re-use by re-shaping""" if name in shared_data_arrays: arg_arr = shared_data_arrays[name] if np.prod(arg_arr.shape) >= np.prod(arg_shape): # nice, we can directly re-use this data blob assert arg_arr.dtype == arg_type arg_arr = arg_arr.reshape(arg_shape) else: logger.warning(('bucketing: data "%s" has a shape %s' % (name, arg_shape)) + (', which is larger than already allocated ') + ('shape %s' % (arg_arr.shape,)) + ('. Need to re-allocate. Consider putting ') + ('default_bucket_key to') + (' be the bucket taking the largest input for better ') + ('memory sharing.')) arg_arr = nd.zeros(arg_shape, context, dtype=arg_type) # replace existing shared array because the new one is bigger shared_data_arrays[name] = arg_arr else: arg_arr = nd.zeros(arg_shape, context, dtype=arg_type) shared_data_arrays[name] = arg_arr return arg_arr # create or borrow arguments and gradients for j in range(len(self.arg_names)): name = self.arg_names[j] if name in self.param_names: # model parameters if shared_exec is None: arg_arr = nd.zeros(arg_shapes[j], context, dtype=arg_types[j]) if self.grad_req[name] != 'null': grad_arr = nd.zeros(arg_shapes[j], context, dtype=arg_types[j]) grad_arrays[name] = grad_arr else: arg_arr = shared_exec.arg_dict[name] assert arg_arr.shape == arg_shapes[j] assert arg_arr.dtype == arg_types[j] if self.grad_req[name] != 'null': grad_arrays[name] = shared_exec.grad_dict[name] else: # data, label, or states arg_arr = _get_or_reshape(name, shared_data_arrays, arg_shapes[j], arg_types[j], context, self.logger) # data might also need grad if inputs_need_grad is True if self.grad_req[name] != 'null': grad_arrays[name] = _get_or_reshape('grad of ' + name, shared_data_arrays, arg_shapes[j], arg_types[j], context, self.logger) arg_arrays.append(arg_arr) # create or borrow aux variables if shared_exec is None: aux_arrays = [nd.zeros(s, context, dtype=t) for s, t in zip(aux_shapes, aux_types)] else: for j, arr in enumerate(shared_exec.aux_arrays): assert aux_shapes[j] == arr.shape assert aux_types[j] == arr.dtype aux_arrays = shared_exec.aux_arrays[:] executor = self.symbol.bind(ctx=context, args=arg_arrays, args_grad=grad_arrays, aux_states=aux_arrays, grad_req=self.grad_req, shared_exec=shared_exec) # Get the total bytes allocated for this executor return executor def _sliced_shape(self, shapes, i, major_axis): """Get the sliced shapes for the i-th executor. Parameters ---------- shapes : list of (str, tuple) The original (name, shape) pairs. i : int Which executor we are dealing with. """ sliced_shapes = [] for desc, axis in zip(shapes, major_axis): shape = list(desc.shape) if axis >= 0: shape[axis] = self.slices[i].stop - self.slices[i].start sliced_shapes.append(DataDesc(desc.name, tuple(shape), desc.dtype, desc.layout)) return sliced_shapes def install_monitor(self, mon): """Install monitor on all executors""" for exe in self.execs: mon.install(exe) ================================================ FILE: dff_rfcn/core/__init__.py ================================================ ================================================ FILE: dff_rfcn/core/callback.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- import time import logging import mxnet as mx class Speedometer(object): def __init__(self, batch_size, frequent=50): self.batch_size = batch_size self.frequent = frequent self.init = False self.tic = 0 self.last_count = 0 def __call__(self, param): """Callback to Show speed.""" count = param.nbatch if self.last_count > count: self.init = False self.last_count = count if self.init: if count % self.frequent == 0: speed = self.frequent * self.batch_size / (time.time() - self.tic) s = '' if param.eval_metric is not None: name, value = param.eval_metric.get() s = "Epoch[%d] Batch [%d]\tSpeed: %.2f samples/sec\tTrain-" % (param.epoch, count, speed) for n, v in zip(name, value): s += "%s=%f,\t" % (n, v) else: s = "Iter[%d] Batch [%d]\tSpeed: %.2f samples/sec" % (param.epoch, count, speed) logging.info(s) print(s) self.tic = time.time() else: self.init = True self.tic = time.time() def do_checkpoint(prefix, means, stds): def _callback(iter_no, sym, arg, aux): weight = arg['rfcn_bbox_weight'] bias = arg['rfcn_bbox_bias'] repeat = bias.shape[0] / means.shape[0] arg['rfcn_bbox_weight_test'] = weight * mx.nd.repeat(mx.nd.array(stds), repeats=repeat).reshape((bias.shape[0], 1, 1, 1)) arg['rfcn_bbox_bias_test'] = arg['rfcn_bbox_bias'] * mx.nd.repeat(mx.nd.array(stds), repeats=repeat) + mx.nd.repeat(mx.nd.array(means), repeats=repeat) mx.model.save_checkpoint(prefix, iter_no + 1, sym, arg, aux) arg.pop('rfcn_bbox_weight_test') arg.pop('rfcn_bbox_bias_test') return _callback ================================================ FILE: dff_rfcn/core/loader.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Xizhou Zhu, Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- import numpy as np import mxnet as mx from mxnet.executor_manager import _split_input_slice from config.config import config from utils.image import tensor_vstack from rpn.rpn import get_rpn_testbatch, get_rpn_pair_batch, assign_anchor from rcnn import get_rcnn_testbatch, get_rcnn_batch class TestLoader(mx.io.DataIter): def __init__(self, roidb, config, batch_size=1, shuffle=False, has_rpn=False): super(TestLoader, self).__init__() # save parameters as properties self.cfg = config self.roidb = roidb self.batch_size = batch_size self.shuffle = shuffle self.has_rpn = has_rpn # infer properties from roidb self.size = np.sum([x['frame_seg_len'] for x in self.roidb]) self.index = np.arange(self.size) # decide data and label names (only for training) self.data_name = ['data', 'im_info', 'data_key', 'feat_key'] self.label_name = None # self.cur_roidb_index = 0 self.cur_frameid = 0 self.data_key = None self.key_frameid = 0 self.cur_seg_len = 0 self.key_frame_flag = -1 # status variable for synchronization between get_data and get_label self.cur = 0 self.data = None self.label = [] self.im_info = None # get first batch to fill in provide_data and provide_label self.reset() self.get_batch() @property def provide_data(self): return [[(k, v.shape) for k, v in zip(self.data_name, idata)] for idata in self.data] @property def provide_label(self): return [None for _ in range(len(self.data))] @property def provide_data_single(self): return [(k, v.shape) for k, v in zip(self.data_name, self.data[0])] @property def provide_label_single(self): return None def reset(self): self.cur = 0 if self.shuffle: np.random.shuffle(self.index) def iter_next(self): return self.cur < self.size def next(self): if self.iter_next(): self.get_batch() self.cur += self.batch_size self.cur_frameid += 1 if self.cur_frameid == self.cur_seg_len: self.cur_roidb_index += 1 self.cur_frameid = 0 self.key_frameid = 0 elif self.cur_frameid - self.key_frameid == self.cfg.TEST.KEY_FRAME_INTERVAL: self.key_frameid = self.cur_frameid return self.im_info, self.key_frame_flag, mx.io.DataBatch(data=self.data, label=self.label, pad=self.getpad(), index=self.getindex(), provide_data=self.provide_data, provide_label=self.provide_label) else: raise StopIteration def getindex(self): return self.cur / self.batch_size def getpad(self): if self.cur + self.batch_size > self.size: return self.cur + self.batch_size - self.size else: return 0 def get_batch(self): cur_roidb = self.roidb[self.cur_roidb_index].copy() cur_roidb['image'] = cur_roidb['pattern'] % self.cur_frameid self.cur_seg_len = cur_roidb['frame_seg_len'] data, label, im_info = get_rpn_testbatch([cur_roidb], self.cfg) if self.key_frameid == self.cur_frameid: # key frame self.data_key = data[0]['data'].copy() if self.key_frameid == 0: self.key_frame_flag = 0 else: self.key_frame_flag = 1 else: self.key_frame_flag = 2 extend_data = [{'data': data[0]['data'], 'im_info': data[0]['im_info'], 'data_key': self.data_key, 'feat_key': np.zeros((1,self.cfg.network.DFF_FEAT_DIM,1,1))}] self.data = [[mx.nd.array(extend_data[i][name]) for name in self.data_name] for i in xrange(len(data))] self.im_info = im_info class AnchorLoader(mx.io.DataIter): def __init__(self, feat_sym, roidb, cfg, batch_size=1, shuffle=False, ctx=None, work_load_list=None, feat_stride=16, anchor_scales=(8, 16, 32), anchor_ratios=(0.5, 1, 2), allowed_border=0, aspect_grouping=False, normalize_target=False, bbox_mean=(0.0, 0.0, 0.0, 0.0), bbox_std=(0.1, 0.1, 0.4, 0.4)): """ This Iter will provide roi data to Fast R-CNN network :param feat_sym: to infer shape of assign_output :param roidb: must be preprocessed :param batch_size: must divide BATCH_SIZE(128) :param shuffle: bool :param ctx: list of contexts :param work_load_list: list of work load :param aspect_grouping: group images with similar aspects :param normalize_target: normalize rpn target :param bbox_mean: anchor target mean :param bbox_std: anchor target std :return: AnchorLoader """ super(AnchorLoader, self).__init__() # save parameters as properties self.feat_sym = feat_sym self.roidb = roidb self.cfg = cfg self.batch_size = batch_size self.shuffle = shuffle self.ctx = ctx if self.ctx is None: self.ctx = [mx.cpu()] self.work_load_list = work_load_list self.feat_stride = feat_stride self.anchor_scales = anchor_scales self.anchor_ratios = anchor_ratios self.allowed_border = allowed_border self.aspect_grouping = aspect_grouping self.normalize_target = normalize_target self.bbox_mean = bbox_mean self.bbox_std = bbox_std # infer properties from roidb self.size = len(roidb) self.index = np.arange(self.size) # decide data and label names if config.TRAIN.END2END: self.data_name = ['data', 'data_ref', 'eq_flag', 'im_info', 'gt_boxes'] else: self.data_name = ['data'] self.label_name = ['label', 'bbox_target', 'bbox_weight'] # status variable for synchronization between get_data and get_label self.cur = 0 self.batch = None self.data = None self.label = None # get first batch to fill in provide_data and provide_label self.reset() self.get_batch_individual() @property def provide_data(self): return [[(k, v.shape) for k, v in zip(self.data_name, self.data[i])] for i in xrange(len(self.data))] @property def provide_label(self): return [[(k, v.shape) for k, v in zip(self.label_name, self.label[i])] for i in xrange(len(self.data))] @property def provide_data_single(self): return [(k, v.shape) for k, v in zip(self.data_name, self.data[0])] @property def provide_label_single(self): return [(k, v.shape) for k, v in zip(self.label_name, self.label[0])] def reset(self): self.cur = 0 if self.shuffle: if self.aspect_grouping: widths = np.array([r['width'] for r in self.roidb]) heights = np.array([r['height'] for r in self.roidb]) horz = (widths >= heights) vert = np.logical_not(horz) horz_inds = np.where(horz)[0] vert_inds = np.where(vert)[0] inds = np.hstack((np.random.permutation(horz_inds), np.random.permutation(vert_inds))) extra = inds.shape[0] % self.batch_size inds_ = np.reshape(inds[:-extra], (-1, self.batch_size)) row_perm = np.random.permutation(np.arange(inds_.shape[0])) inds[:-extra] = np.reshape(inds_[row_perm, :], (-1,)) self.index = inds else: np.random.shuffle(self.index) def iter_next(self): return self.cur + self.batch_size <= self.size def next(self): if self.iter_next(): self.get_batch_individual() self.cur += self.batch_size return mx.io.DataBatch(data=self.data, label=self.label, pad=self.getpad(), index=self.getindex(), provide_data=self.provide_data, provide_label=self.provide_label) else: raise StopIteration def getindex(self): return self.cur / self.batch_size def getpad(self): if self.cur + self.batch_size > self.size: return self.cur + self.batch_size - self.size else: return 0 def infer_shape(self, max_data_shape=None, max_label_shape=None): """ Return maximum data and label shape for single gpu """ if max_data_shape is None: max_data_shape = [] if max_label_shape is None: max_label_shape = [] max_shapes = dict(max_data_shape + max_label_shape) input_batch_size = max_shapes['data'][0] im_info = [[max_shapes['data'][2], max_shapes['data'][3], 1.0]] _, feat_shape, _ = self.feat_sym.infer_shape(**max_shapes) label = assign_anchor(feat_shape[0], np.zeros((0, 5)), im_info, self.cfg, self.feat_stride, self.anchor_scales, self.anchor_ratios, self.allowed_border, self.normalize_target, self.bbox_mean, self.bbox_std) label = [label[k] for k in self.label_name] label_shape = [(k, tuple([input_batch_size] + list(v.shape[1:]))) for k, v in zip(self.label_name, label)] return max_data_shape, label_shape def get_batch(self): # slice roidb cur_from = self.cur cur_to = min(cur_from + self.batch_size, self.size) roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)] # decide multi device slice work_load_list = self.work_load_list ctx = self.ctx if work_load_list is None: work_load_list = [1] * len(ctx) assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \ "Invalid settings for work load. " slices = _split_input_slice(self.batch_size, work_load_list) # get testing data for multigpu data_list = [] label_list = [] for islice in slices: iroidb = [roidb[i] for i in range(islice.start, islice.stop)] data, label = get_rpn_pair_batch(iroidb, self.cfg) data_list.append(data) label_list.append(label) # pad data first and then assign anchor (read label) data_tensor = tensor_vstack([batch['data'] for batch in data_list]) for data, data_pad in zip(data_list, data_tensor): data['data'] = data_pad[np.newaxis, :] new_label_list = [] for data, label in zip(data_list, label_list): # infer label shape data_shape = {k: v.shape for k, v in data.items()} del data_shape['im_info'] _, feat_shape, _ = self.feat_sym.infer_shape(**data_shape) feat_shape = [int(i) for i in feat_shape[0]] # add gt_boxes to data for e2e data['gt_boxes'] = label['gt_boxes'][np.newaxis, :, :] # assign anchor for label label = assign_anchor(feat_shape, label['gt_boxes'], data['im_info'], self.cfg, self.feat_stride, self.anchor_scales, self.anchor_ratios, self.allowed_border, self.normalize_target, self.bbox_mean, self.bbox_std) new_label_list.append(label) all_data = dict() for key in self.data_name: all_data[key] = tensor_vstack([batch[key] for batch in data_list]) all_label = dict() for key in self.label_name: pad = -1 if key == 'label' else 0 all_label[key] = tensor_vstack([batch[key] for batch in new_label_list], pad=pad) self.data = [mx.nd.array(all_data[key]) for key in self.data_name] self.label = [mx.nd.array(all_label[key]) for key in self.label_name] def get_batch_individual(self): cur_from = self.cur cur_to = min(cur_from + self.batch_size, self.size) roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)] # decide multi device slice work_load_list = self.work_load_list ctx = self.ctx if work_load_list is None: work_load_list = [1] * len(ctx) assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \ "Invalid settings for work load. " slices = _split_input_slice(self.batch_size, work_load_list) rst = [] for idx, islice in enumerate(slices): iroidb = [roidb[i] for i in range(islice.start, islice.stop)] rst.append(self.parfetch(iroidb)) all_data = [_['data'] for _ in rst] all_label = [_['label'] for _ in rst] self.data = [[mx.nd.array(data[key]) for key in self.data_name] for data in all_data] self.label = [[mx.nd.array(label[key]) for key in self.label_name] for label in all_label] def parfetch(self, iroidb): # get testing data for multigpu data, label = get_rpn_pair_batch(iroidb, self.cfg) data_shape = {k: v.shape for k, v in data.items()} del data_shape['im_info'] _, feat_shape, _ = self.feat_sym.infer_shape(**data_shape) feat_shape = [int(i) for i in feat_shape[0]] # add gt_boxes to data for e2e data['gt_boxes'] = label['gt_boxes'][np.newaxis, :, :] # assign anchor for label label = assign_anchor(feat_shape, label['gt_boxes'], data['im_info'], self.cfg, self.feat_stride, self.anchor_scales, self.anchor_ratios, self.allowed_border, self.normalize_target, self.bbox_mean, self.bbox_std) return {'data': data, 'label': label} ================================================ FILE: dff_rfcn/core/metric.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- import mxnet as mx import numpy as np def get_rpn_names(): pred = ['rpn_cls_prob', 'rpn_bbox_loss'] label = ['rpn_label', 'rpn_bbox_target', 'rpn_bbox_weight'] return pred, label def get_rcnn_names(cfg): pred = ['rcnn_cls_prob', 'rcnn_bbox_loss'] label = ['rcnn_label', 'rcnn_bbox_target', 'rcnn_bbox_weight'] if cfg.TRAIN.ENABLE_OHEM or cfg.TRAIN.END2END: pred.append('rcnn_label') if cfg.TRAIN.END2END: rpn_pred, rpn_label = get_rpn_names() pred = rpn_pred + pred label = rpn_label return pred, label class RPNAccMetric(mx.metric.EvalMetric): def __init__(self): super(RPNAccMetric, self).__init__('RPNAcc') self.pred, self.label = get_rpn_names() def update(self, labels, preds): pred = preds[self.pred.index('rpn_cls_prob')] label = labels[self.label.index('rpn_label')] # pred (b, c, p) or (b, c, h, w) pred_label = mx.ndarray.argmax_channel(pred).asnumpy().astype('int32') pred_label = pred_label.reshape((pred_label.shape[0], -1)) # label (b, p) label = label.asnumpy().astype('int32') # filter with keep_inds keep_inds = np.where(label != -1) pred_label = pred_label[keep_inds] label = label[keep_inds] self.sum_metric += np.sum(pred_label.flat == label.flat) self.num_inst += len(pred_label.flat) class RCNNAccMetric(mx.metric.EvalMetric): def __init__(self, cfg): super(RCNNAccMetric, self).__init__('RCNNAcc') self.e2e = cfg.TRAIN.END2END self.ohem = cfg.TRAIN.ENABLE_OHEM self.pred, self.label = get_rcnn_names(cfg) def update(self, labels, preds): pred = preds[self.pred.index('rcnn_cls_prob')] if self.ohem or self.e2e: label = preds[self.pred.index('rcnn_label')] else: label = labels[self.label.index('rcnn_label')] last_dim = pred.shape[-1] pred_label = pred.asnumpy().reshape(-1, last_dim).argmax(axis=1).astype('int32') label = label.asnumpy().reshape(-1,).astype('int32') # filter with keep_inds keep_inds = np.where(label != -1) pred_label = pred_label[keep_inds] label = label[keep_inds] self.sum_metric += np.sum(pred_label.flat == label.flat) self.num_inst += len(pred_label.flat) class RPNLogLossMetric(mx.metric.EvalMetric): def __init__(self): super(RPNLogLossMetric, self).__init__('RPNLogLoss') self.pred, self.label = get_rpn_names() def update(self, labels, preds): pred = preds[self.pred.index('rpn_cls_prob')] label = labels[self.label.index('rpn_label')] # label (b, p) label = label.asnumpy().astype('int32').reshape((-1)) # pred (b, c, p) or (b, c, h, w) --> (b, p, c) --> (b*p, c) pred = pred.asnumpy().reshape((pred.shape[0], pred.shape[1], -1)).transpose((0, 2, 1)) pred = pred.reshape((label.shape[0], -1)) # filter with keep_inds keep_inds = np.where(label != -1)[0] label = label[keep_inds] cls = pred[keep_inds, label] cls += 1e-14 cls_loss = -1 * np.log(cls) cls_loss = np.sum(cls_loss) self.sum_metric += cls_loss self.num_inst += label.shape[0] class RCNNLogLossMetric(mx.metric.EvalMetric): def __init__(self, cfg): super(RCNNLogLossMetric, self).__init__('RCNNLogLoss') self.e2e = cfg.TRAIN.END2END self.ohem = cfg.TRAIN.ENABLE_OHEM self.pred, self.label = get_rcnn_names(cfg) def update(self, labels, preds): pred = preds[self.pred.index('rcnn_cls_prob')] if self.ohem or self.e2e: label = preds[self.pred.index('rcnn_label')] else: label = labels[self.label.index('rcnn_label')] last_dim = pred.shape[-1] pred = pred.asnumpy().reshape(-1, last_dim) label = label.asnumpy().reshape(-1,).astype('int32') # filter with keep_inds keep_inds = np.where(label != -1)[0] label = label[keep_inds] cls = pred[keep_inds, label] cls += 1e-14 cls_loss = -1 * np.log(cls) cls_loss = np.sum(cls_loss) self.sum_metric += cls_loss self.num_inst += label.shape[0] class RPNL1LossMetric(mx.metric.EvalMetric): def __init__(self): super(RPNL1LossMetric, self).__init__('RPNL1Loss') self.pred, self.label = get_rpn_names() def update(self, labels, preds): bbox_loss = preds[self.pred.index('rpn_bbox_loss')].asnumpy() # calculate num_inst (average on those kept anchors) label = labels[self.label.index('rpn_label')].asnumpy() num_inst = np.sum(label != -1) self.sum_metric += np.sum(bbox_loss) self.num_inst += num_inst class RCNNL1LossMetric(mx.metric.EvalMetric): def __init__(self, cfg): super(RCNNL1LossMetric, self).__init__('RCNNL1Loss') self.e2e = cfg.TRAIN.END2END self.ohem = cfg.TRAIN.ENABLE_OHEM self.pred, self.label = get_rcnn_names(cfg) def update(self, labels, preds): bbox_loss = preds[self.pred.index('rcnn_bbox_loss')].asnumpy() if self.ohem: label = preds[self.pred.index('rcnn_label')].asnumpy() else: if self.e2e: label = preds[self.pred.index('rcnn_label')].asnumpy() else: label = labels[self.label.index('rcnn_label')].asnumpy() # calculate num_inst (average on those kept anchors) num_inst = np.sum(label != -1) self.sum_metric += np.sum(bbox_loss) self.num_inst += num_inst ================================================ FILE: dff_rfcn/core/module.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- """A `MutableModule` implement the `BaseModule` API, and allows input shape varying with training iterations. If shapes vary, executors will rebind, using shared arrays from the initial module binded with maximum shape. """ import time import logging import warnings from mxnet import context as ctx from mxnet.initializer import Uniform, InitDesc from mxnet.module.base_module import BaseModule, _check_input_names, _parse_data_desc, _as_list from mxnet.model import _create_kvstore, _initialize_kvstore, _update_params, _update_params_on_kvstore, load_checkpoint, BatchEndParam from mxnet import metric from .DataParallelExecutorGroup import DataParallelExecutorGroup from mxnet import ndarray as nd from mxnet import optimizer as opt class Module(BaseModule): """Module is a basic module that wrap a `Symbol`. It is functionally the same as the `FeedForward` model, except under the module API. Parameters ---------- symbol : Symbol data_names : list of str Default is `('data')` for a typical model used in image classification. label_names : list of str Default is `('softmax_label')` for a typical model used in image classification. logger : Logger Default is `logging`. context : Context or list of Context Default is `cpu()`. work_load_list : list of number Default `None`, indicating uniform workload. fixed_param_names: list of str Default `None`, indicating no network parameters are fixed. state_names : list of str states are similar to data and label, but not provided by data iterator. Instead they are initialized to 0 and can be set by set_states() """ def __init__(self, symbol, data_names=('data',), label_names=('softmax_label',), logger=logging, context=ctx.cpu(), work_load_list=None, fixed_param_names=None, state_names=None): super(Module, self).__init__(logger=logger) if isinstance(context, ctx.Context): context = [context] self._context = context if work_load_list is None: work_load_list = [1] * len(self._context) assert len(work_load_list) == len(self._context) self._work_load_list = work_load_list self._symbol = symbol data_names = list(data_names) if data_names is not None else [] label_names = list(label_names) if label_names is not None else [] state_names = list(state_names) if state_names is not None else [] fixed_param_names = list(fixed_param_names) if fixed_param_names is not None else [] _check_input_names(symbol, data_names, "data", True) _check_input_names(symbol, label_names, "label", False) _check_input_names(symbol, state_names, "state", True) _check_input_names(symbol, fixed_param_names, "fixed_param", True) arg_names = symbol.list_arguments() input_names = data_names + label_names + state_names self._param_names = [x for x in arg_names if x not in input_names] self._fixed_param_names = fixed_param_names self._aux_names = symbol.list_auxiliary_states() self._data_names = data_names self._label_names = label_names self._state_names = state_names self._output_names = symbol.list_outputs() self._arg_params = None self._aux_params = None self._params_dirty = False self._optimizer = None self._kvstore = None self._update_on_kvstore = None self._updater = None self._preload_opt_states = None self._grad_req = None self._exec_group = None self._data_shapes = None self._label_shapes = None @staticmethod def load(prefix, epoch, load_optimizer_states=False, **kwargs): """Create a model from previously saved checkpoint. Parameters ---------- prefix : str path prefix of saved model files. You should have "prefix-symbol.json", "prefix-xxxx.params", and optionally "prefix-xxxx.states", where xxxx is the epoch number. epoch : int epoch to load. load_optimizer_states : bool whether to load optimizer states. Checkpoint needs to have been made with save_optimizer_states=True. data_names : list of str Default is `('data')` for a typical model used in image classification. label_names : list of str Default is `('softmax_label')` for a typical model used in image classification. logger : Logger Default is `logging`. context : Context or list of Context Default is `cpu()`. work_load_list : list of number Default `None`, indicating uniform workload. fixed_param_names: list of str Default `None`, indicating no network parameters are fixed. """ sym, args, auxs = load_checkpoint(prefix, epoch) mod = Module(symbol=sym, **kwargs) mod._arg_params = args mod._aux_params = auxs mod.params_initialized = True if load_optimizer_states: mod._preload_opt_states = '%s-%04d.states'%(prefix, epoch) return mod def save_checkpoint(self, prefix, epoch, save_optimizer_states=False): """Save current progress to checkpoint. Use mx.callback.module_checkpoint as epoch_end_callback to save during training. Parameters ---------- prefix : str The file prefix to checkpoint to epoch : int The current epoch number save_optimizer_states : bool Whether to save optimizer states for continue training """ self._symbol.save('%s-symbol.json'%prefix) param_name = '%s-%04d.params' % (prefix, epoch) self.save_params(param_name) logging.info('Saved checkpoint to \"%s\"', param_name) if save_optimizer_states: state_name = '%s-%04d.states' % (prefix, epoch) self.save_optimizer_states(state_name) logging.info('Saved optimizer state to \"%s\"', state_name) def _reset_bind(self): """Internal function to reset binded state.""" self.binded = False self._exec_group = None self._data_shapes = None self._label_shapes = None @property def data_names(self): """A list of names for data required by this module.""" return self._data_names @property def label_names(self): """A list of names for labels required by this module.""" return self._label_names @property def output_names(self): """A list of names for the outputs of this module.""" return self._output_names @property def data_shapes(self): """Get data shapes. Returns ------- A list of `(name, shape)` pairs. """ assert self.binded return self._data_shapes @property def label_shapes(self): """Get label shapes. Returns ------- A list of `(name, shape)` pairs. The return value could be `None` if the module does not need labels, or if the module is not binded for training (in this case, label information is not available). """ assert self.binded return self._label_shapes @property def output_shapes(self): """Get output shapes. Returns ------- A list of `(name, shape)` pairs. """ assert self.binded return self._exec_group.get_output_shapes() def get_params(self): """Get current parameters. Returns ------- `(arg_params, aux_params)`, each a dictionary of name to parameters (in `NDArray`) mapping. """ assert self.binded and self.params_initialized if self._params_dirty: self._sync_params_from_devices() return (self._arg_params, self._aux_params) def init_params(self, initializer=Uniform(0.01), arg_params=None, aux_params=None, allow_missing=False, force_init=False): """Initialize the parameters and auxiliary states. Parameters ---------- initializer : Initializer Called to initialize parameters if needed. arg_params : dict If not None, should be a dictionary of existing arg_params. Initialization will be copied from that. aux_params : dict If not None, should be a dictionary of existing aux_params. Initialization will be copied from that. allow_missing : bool If true, params could contain missing values, and the initializer will be called to fill those missing params. force_init : bool If true, will force re-initialize even if already initialized. """ if self.params_initialized and not force_init: warnings.warn("Parameters already initialized and force_init=False. " "init_params call ignored.", stacklevel=2) return assert self.binded, 'call bind before initializing the parameters' def _impl(name, arr, cache): """Internal helper for parameter initialization""" if cache is not None: if name in cache: cache_arr = cache[name] # just in case the cached array is just the target itself if cache_arr is not arr: cache_arr.copyto(arr) else: if not allow_missing: raise RuntimeError("%s is not presented" % name) if initializer != None: initializer(name, arr) else: initializer(name, arr) attrs = self._symbol.attr_dict() for name, arr in self._arg_params.items(): desc = InitDesc(name, attrs.get(name, None)) _impl(desc, arr, arg_params) for name, arr in self._aux_params.items(): desc = InitDesc(name, attrs.get(name, None)) _impl(desc, arr, aux_params) self.params_initialized = True self._params_dirty = False # copy the initialized parameters to devices self._exec_group.set_params(self._arg_params, self._aux_params) def set_params(self, arg_params, aux_params, allow_missing=False, force_init=True): """Assign parameter and aux state values. Parameters ---------- arg_params : dict Dictionary of name to value (`NDArray`) mapping. aux_params : dict Dictionary of name to value (`NDArray`) mapping. allow_missing : bool If true, params could contain missing values, and the initializer will be called to fill those missing params. force_init : bool If true, will force re-initialize even if already initialized. Examples -------- An example of setting module parameters:: >>> sym, arg_params, aux_params = \ >>> mx.model.load_checkpoint(model_prefix, n_epoch_load) >>> mod.set_params(arg_params=arg_params, aux_params=aux_params) """ if not allow_missing: self.init_params(initializer=None, arg_params=arg_params, aux_params=aux_params, allow_missing=allow_missing, force_init=force_init) return if self.params_initialized and not force_init: warnings.warn("Parameters already initialized and force_init=False. " "set_params call ignored.", stacklevel=2) return self._exec_group.set_params(arg_params, aux_params) # because we didn't update self._arg_params, they are dirty now. self._params_dirty = True self.params_initialized = True def bind(self, data_shapes, label_shapes=None, for_training=True, inputs_need_grad=False, force_rebind=False, shared_module=None, grad_req='write'): """Bind the symbols to construct executors. This is necessary before one can perform computation with the module. Parameters ---------- data_shapes : list of (str, tuple) Typically is `data_iter.provide_data`. label_shapes : list of (str, tuple) Typically is `data_iter.provide_label`. for_training : bool Default is `True`. Whether the executors should be bind for training. inputs_need_grad : bool Default is `False`. Whether the gradients to the input data need to be computed. Typically this is not needed. But this might be needed when implementing composition of modules. force_rebind : bool Default is `False`. This function does nothing if the executors are already binded. But with this `True`, the executors will be forced to rebind. shared_module : Module Default is `None`. This is used in bucketing. When not `None`, the shared module essentially corresponds to a different bucket -- a module with different symbol but with the same sets of parameters (e.g. unrolled RNNs with different lengths). """ # force rebinding is typically used when one want to switch from # training to prediction phase. if force_rebind: self._reset_bind() if self.binded: self.logger.warning('Already binded, ignoring bind()') return self.for_training = for_training self.inputs_need_grad = inputs_need_grad self.binded = True self._grad_req = grad_req if not for_training: assert not inputs_need_grad else: pass # this is not True, as some module might not contains a loss function # that consumes the labels # assert label_shapes is not None # self._data_shapes, self._label_shapes = _parse_data_desc( # self.data_names, self.label_names, data_shapes, label_shapes) self._data_shapes, self._label_shapes = zip(*[_parse_data_desc(self.data_names, self.label_names, data_shape, label_shape) for data_shape, label_shape in zip(data_shapes, label_shapes)]) if self._label_shapes.count(None) == len(self._label_shapes): self._label_shapes = None if shared_module is not None: assert isinstance(shared_module, Module) and \ shared_module.binded and shared_module.params_initialized shared_group = shared_module._exec_group else: shared_group = None self._exec_group = DataParallelExecutorGroup(self._symbol, self._context, self._work_load_list, self._data_shapes, self._label_shapes, self._param_names, for_training, inputs_need_grad, shared_group, logger=self.logger, fixed_param_names=self._fixed_param_names, grad_req=grad_req, state_names=self._state_names) # self._total_exec_bytes = self._exec_group._total_exec_bytes if shared_module is not None: self.params_initialized = True self._arg_params = shared_module._arg_params self._aux_params = shared_module._aux_params elif self.params_initialized: # if the parameters are already initialized, we are re-binding # so automatically copy the already initialized params self._exec_group.set_params(self._arg_params, self._aux_params) else: assert self._arg_params is None and self._aux_params is None param_arrays = [ nd.zeros(x[0].shape, dtype=x[0].dtype) for x in self._exec_group.param_arrays ] self._arg_params = {name:arr for name, arr in zip(self._param_names, param_arrays)} aux_arrays = [ nd.zeros(x[0].shape, dtype=x[0].dtype) for x in self._exec_group.aux_arrays ] self._aux_params = {name:arr for name, arr in zip(self._aux_names, aux_arrays)} if shared_module is not None and shared_module.optimizer_initialized: self.borrow_optimizer(shared_module) def reshape(self, data_shapes, label_shapes=None): """Reshape the module for new input shapes. Parameters ---------- data_shapes : list of (str, tuple) Typically is `data_iter.provide_data`. label_shapes : list of (str, tuple) Typically is `data_iter.provide_label`. """ assert self.binded # self._data_shapes, self._label_shapes = _parse_data_desc( # self.data_names, self.label_names, data_shapes, label_shapes) self._data_shapes, self._label_shapes = zip(*[_parse_data_desc(self.data_names, self.label_names, data_shape, label_shape) for data_shape, label_shape in zip(data_shapes, label_shapes)]) self._exec_group.reshape(self._data_shapes, self._label_shapes) def init_optimizer(self, kvstore='local', optimizer='sgd', optimizer_params=(('learning_rate', 0.01),), force_init=False): """Install and initialize optimizers. Parameters ---------- kvstore : str or KVStore Default `'local'`. optimizer : str or Optimizer Default `'sgd'` optimizer_params : dict Default `(('learning_rate', 0.01),)`. The default value is not a dictionary, just to avoid pylint warning of dangerous default values. force_init : bool Default `False`, indicating whether we should force re-initializing the optimizer in the case an optimizer is already installed. """ assert self.binded and self.params_initialized if self.optimizer_initialized and not force_init: self.logger.warning('optimizer already initialized, ignoring...') return (kvstore, update_on_kvstore) = \ _create_kvstore(kvstore, len(self._context), self._arg_params) batch_size = self._exec_group.batch_size if kvstore and 'dist' in kvstore.type and '_sync' in kvstore.type: batch_size *= kvstore.num_workers rescale_grad = 1.0/batch_size if isinstance(optimizer, str): idx2name = {} if update_on_kvstore: idx2name.update(enumerate(self._exec_group.param_names)) else: for k in range(len(self._context)): idx2name.update({i*len(self._context)+k: n for i, n in enumerate(self._exec_group.param_names)}) optimizer_params = dict(optimizer_params) if 'rescale_grad' not in optimizer_params: optimizer_params['rescale_grad'] = rescale_grad optimizer = opt.create(optimizer, sym=self.symbol, param_idx2name=idx2name, **optimizer_params) else: assert isinstance(optimizer, opt.Optimizer) if optimizer.rescale_grad != rescale_grad: #pylint: disable=no-member warnings.warn( "Optimizer created manually outside Module but rescale_grad " + "is not normalized to 1.0/batch_size/num_workers (%s vs. %s). "%( optimizer.rescale_grad, rescale_grad) + "Is this intended?", stacklevel=2) self._optimizer = optimizer self._kvstore = kvstore self._update_on_kvstore = update_on_kvstore self._updater = None if kvstore: # copy initialized local parameters to kvstore _initialize_kvstore(kvstore=kvstore, param_arrays=self._exec_group.param_arrays, arg_params=self._arg_params, param_names=self._param_names, update_on_kvstore=update_on_kvstore) if update_on_kvstore: kvstore.set_optimizer(self._optimizer) else: self._updater = opt.get_updater(optimizer) self.optimizer_initialized = True if self._preload_opt_states is not None: self.load_optimizer_states(self._preload_opt_states) self._preload_opt_states = None def borrow_optimizer(self, shared_module): """Borrow optimizer from a shared module. Used in bucketing, where exactly the same optimizer (esp. kvstore) is used. Parameters ---------- shared_module : Module """ assert shared_module.optimizer_initialized self._optimizer = shared_module._optimizer self._kvstore = shared_module._kvstore self._update_on_kvstore = shared_module._update_on_kvstore self._updater = shared_module._updater self.optimizer_initialized = True def forward(self, data_batch, is_train=None): """Forward computation. Parameters ---------- data_batch : DataBatch Could be anything with similar API implemented. is_train : bool Default is `None`, which means `is_train` takes the value of `self.for_training`. """ assert self.binded and self.params_initialized self._exec_group.forward(data_batch, is_train) def backward(self, out_grads=None): """Backward computation. Parameters ---------- out_grads : NDArray or list of NDArray, optional Gradient on the outputs to be propagated back. This parameter is only needed when bind is called on outputs that are not a loss function. """ assert self.binded and self.params_initialized self._exec_group.backward(out_grads=out_grads) def update(self): """Update parameters according to the installed optimizer and the gradients computed in the previous forward-backward batch. """ assert self.binded and self.params_initialized and self.optimizer_initialized self._params_dirty = True if self._update_on_kvstore: _update_params_on_kvstore(self._exec_group.param_arrays, self._exec_group.grad_arrays, self._kvstore) else: _update_params(self._exec_group.param_arrays, self._exec_group.grad_arrays, updater=self._updater, num_device=len(self._context), kvstore=self._kvstore) def get_outputs(self, merge_multi_context=True): """Get outputs of the previous forward computation. Parameters ---------- merge_multi_context : bool Default is `True`. In the case when data-parallelism is used, the outputs will be collected from multiple devices. A `True` value indicate that we should merge the collected results so that they look like from a single executor. Returns ------- If `merge_multi_context` is `True`, it is like `[out1, out2]`. Otherwise, it is like `[[out1_dev1, out1_dev2], [out2_dev1, out2_dev2]]`. All the output elements are `NDArray`. """ assert self.binded and self.params_initialized return self._exec_group.get_outputs(merge_multi_context=merge_multi_context) def get_input_grads(self, merge_multi_context=True): """Get the gradients with respect to the inputs of the module. Parameters ---------- merge_multi_context : bool Default is `True`. In the case when data-parallelism is used, the outputs will be collected from multiple devices. A `True` value indicate that we should merge the collected results so that they look like from a single executor. Returns ------- If `merge_multi_context` is `True`, it is like `[grad1, grad2]`. Otherwise, it is like `[[grad1_dev1, grad1_dev2], [grad2_dev1, grad2_dev2]]`. All the output elements are `NDArray`. """ assert self.binded and self.params_initialized and self.inputs_need_grad return self._exec_group.get_input_grads(merge_multi_context=merge_multi_context) def get_states(self, merge_multi_context=True): """Get states from all devices Parameters ---------- merge_multi_context : bool Default is `True`. In the case when data-parallelism is used, the states will be collected from multiple devices. A `True` value indicate that we should merge the collected results so that they look like from a single executor. Returns ------- If `merge_multi_context` is `True`, it is like `[out1, out2]`. Otherwise, it is like `[[out1_dev1, out1_dev2], [out2_dev1, out2_dev2]]`. All the output elements are `NDArray`. """ assert self.binded and self.params_initialized return self._exec_group.get_states(merge_multi_context=merge_multi_context) def set_states(self, states=None, value=None): """Set value for states. Only one of states & value can be specified. Parameters ---------- states : list of list of NDArrays source states arrays formatted like [[state1_dev1, state1_dev2], [state2_dev1, state2_dev2]]. value : number a single scalar value for all state arrays. """ assert self.binded and self.params_initialized self._exec_group.set_states(states, value) def update_metric(self, eval_metric, labels): """Evaluate and accumulate evaluation metric on outputs of the last forward computation. Parameters ---------- eval_metric : EvalMetric labels : list of NDArray Typically `data_batch.label`. """ self._exec_group.update_metric(eval_metric, labels) def _sync_params_from_devices(self): """Synchronize parameters from devices to CPU. This function should be called after calling `update` that updates the parameters on the devices, before one can read the latest parameters from `self._arg_params` and `self._aux_params`. """ self._exec_group.get_params(self._arg_params, self._aux_params) self._params_dirty = False def save_optimizer_states(self, fname): """Save optimizer (updater) state to file Parameters ---------- fname : str Path to output states file. """ assert self.optimizer_initialized if self._update_on_kvstore: self._kvstore.save_optimizer_states(fname) else: with open(fname, 'wb') as fout: fout.write(self._updater.get_states()) def load_optimizer_states(self, fname): """Load optimizer (updater) state from file Parameters ---------- fname : str Path to input states file. """ assert self.optimizer_initialized if self._update_on_kvstore: self._kvstore.load_optimizer_states(fname) else: self._updater.set_states(open(fname, 'rb').read()) def install_monitor(self, mon): """ Install monitor on all executors """ assert self.binded self._exec_group.install_monitor(mon) class MutableModule(BaseModule): """A mutable module is a module that supports variable input data. Parameters ---------- symbol : Symbol data_names : list of str label_names : list of str logger : Logger context : Context or list of Context work_load_list : list of number max_data_shapes : list of (name, shape) tuple, designating inputs whose shape vary max_label_shapes : list of (name, shape) tuple, designating inputs whose shape vary fixed_param_prefix : list of str, indicating fixed parameters """ def __init__(self, symbol, data_names, label_names, logger=logging, context=ctx.cpu(), work_load_list=None, max_data_shapes=None, max_label_shapes=None, fixed_param_prefix=None): super(MutableModule, self).__init__(logger=logger) self._symbol = symbol self._data_names = data_names self._label_names = label_names self._context = context self._work_load_list = work_load_list self._curr_module = None self._max_data_shapes = max_data_shapes self._max_label_shapes = max_label_shapes self._fixed_param_prefix = fixed_param_prefix fixed_param_names = list() if fixed_param_prefix is not None: for name in self._symbol.list_arguments(): for prefix in self._fixed_param_prefix: if name.startswith(prefix): fixed_param_names.append(name) self._fixed_param_names = fixed_param_names self._preload_opt_states = None def _reset_bind(self): self.binded = False self._curr_module = None @property def data_names(self): return self._data_names @property def output_names(self): return self._symbol.list_outputs() @property def data_shapes(self): assert self.binded return self._curr_module.data_shapes @property def label_shapes(self): assert self.binded return self._curr_module.label_shapes @property def output_shapes(self): assert self.binded return self._curr_module.output_shapes def get_params(self): assert self.binded and self.params_initialized return self._curr_module.get_params() def init_params(self, initializer=Uniform(0.01), arg_params=None, aux_params=None, allow_missing=False, force_init=False): if self.params_initialized and not force_init: return assert self.binded, 'call bind before initializing the parameters' self._curr_module.init_params(initializer=initializer, arg_params=arg_params, aux_params=aux_params, allow_missing=allow_missing, force_init=force_init) self.params_initialized = True def bind(self, data_shapes, label_shapes=None, for_training=True, inputs_need_grad=False, force_rebind=False, shared_module=None, grad_req='write'): # in case we already initialized params, keep it if self.params_initialized: arg_params, aux_params = self.get_params() # force rebinding is typically used when one want to switch from # training to prediction phase. if force_rebind: self._reset_bind() if self.binded: self.logger.warning('Already binded, ignoring bind()') return assert shared_module is None, 'shared_module for MutableModule is not supported' self.for_training = for_training self.inputs_need_grad = inputs_need_grad self.binded = True max_shapes_dict = dict() if self._max_data_shapes is not None: max_shapes_dict.update(dict(self._max_data_shapes[0])) if self._max_label_shapes is not None: max_shapes_dict.update(dict(self._max_label_shapes[0])) max_data_shapes = list() for name, shape in data_shapes[0]: if name in max_shapes_dict: max_data_shapes.append((name, max_shapes_dict[name])) else: max_data_shapes.append((name, shape)) max_label_shapes = list() if not label_shapes.count(None) == len(label_shapes): for name, shape in label_shapes[0]: if name in max_shapes_dict: max_label_shapes.append((name, max_shapes_dict[name])) else: max_label_shapes.append((name, shape)) if len(max_label_shapes) == 0: max_label_shapes = None module = Module(self._symbol, self._data_names, self._label_names, logger=self.logger, context=self._context, work_load_list=self._work_load_list, fixed_param_names=self._fixed_param_names) module.bind([max_data_shapes for _ in xrange(len(self._context))], [max_label_shapes for _ in xrange(len(self._context))], for_training, inputs_need_grad, force_rebind=False, shared_module=None) self._curr_module = module # copy back saved params, if already initialized if self.params_initialized: self.set_params(arg_params, aux_params) def save_checkpoint(self, prefix, epoch, save_optimizer_states=False): """Save current progress to checkpoint. Use mx.callback.module_checkpoint as epoch_end_callback to save during training. Parameters ---------- prefix : str The file prefix to checkpoint to epoch : int The current epoch number save_optimizer_states : bool Whether to save optimizer states for continue training """ self._curr_module.save_checkpoint(prefix, epoch, save_optimizer_states) def init_optimizer(self, kvstore='local', optimizer='sgd', optimizer_params=(('learning_rate', 0.01),), force_init=False): assert self.binded and self.params_initialized if self.optimizer_initialized and not force_init: self.logger.warning('optimizer already initialized, ignoring.') return self._curr_module._preload_opt_states = self._preload_opt_states self._curr_module.init_optimizer(kvstore, optimizer, optimizer_params, force_init=force_init) self.optimizer_initialized = True def fit(self, train_data, eval_data=None, eval_metric='acc', epoch_end_callback=None, batch_end_callback=None, kvstore='local', optimizer='sgd', optimizer_params=(('learning_rate', 0.01),), eval_end_callback=None, eval_batch_end_callback=None, initializer=Uniform(0.01), arg_params=None, aux_params=None, allow_missing=False, force_rebind=False, force_init=False, begin_epoch=0, num_epoch=None, validation_metric=None, monitor=None, prefix=None): """Train the module parameters. Parameters ---------- train_data : DataIter eval_data : DataIter If not `None`, will be used as validation set and evaluate the performance after each epoch. eval_metric : str or EvalMetric Default `'acc'`. The performance measure used to display during training. epoch_end_callback : function or list of function Each callback will be called with the current `epoch`, `symbol`, `arg_params` and `aux_params`. batch_end_callback : function or list of function Each callback will be called with a `BatchEndParam`. kvstore : str or KVStore Default `'local'`. optimizer : str or Optimizer Default `'sgd'` optimizer_params : dict Default `(('learning_rate', 0.01),)`. The parameters for the optimizer constructor. The default value is not a `dict`, just to avoid pylint warning on dangerous default values. eval_end_callback : function or list of function These will be called at the end of each full evaluation, with the metrics over the entire evaluation set. eval_batch_end_callback : function or list of function These will be called at the end of each minibatch during evaluation initializer : Initializer Will be called to initialize the module parameters if not already initialized. arg_params : dict Default `None`, if not `None`, should be existing parameters from a trained model or loaded from a checkpoint (previously saved model). In this case, the value here will be used to initialize the module parameters, unless they are already initialized by the user via a call to `init_params` or `fit`. `arg_params` has higher priority to `initializer`. aux_params : dict Default `None`. Similar to `arg_params`, except for auxiliary states. allow_missing : bool Default `False`. Indicate whether we allow missing parameters when `arg_params` and `aux_params` are not `None`. If this is `True`, then the missing parameters will be initialized via the `initializer`. force_rebind : bool Default `False`. Whether to force rebinding the executors if already binded. force_init : bool Default `False`. Indicate whether we should force initialization even if the parameters are already initialized. begin_epoch : int Default `0`. Indicate the starting epoch. Usually, if we are resuming from a checkpoint saved at a previous training phase at epoch N, then we should specify this value as N+1. num_epoch : int Number of epochs to run training. Examples -------- An example of using fit for training:: >>> #Assume training dataIter and validation dataIter are ready >>> mod.fit(train_data=train_dataiter, eval_data=val_dataiter, optimizer_params={'learning_rate':0.01, 'momentum': 0.9}, num_epoch=10) """ assert num_epoch is not None, 'please specify number of epochs' self.bind(data_shapes=train_data.provide_data, label_shapes=train_data.provide_label, for_training=True, force_rebind=force_rebind) if monitor is not None: self.install_monitor(monitor) self.init_params(initializer=initializer, arg_params=arg_params, aux_params=aux_params, allow_missing=allow_missing, force_init=force_init) self.init_optimizer(kvstore=kvstore, optimizer=optimizer, optimizer_params=optimizer_params) if validation_metric is None: validation_metric = eval_metric if not isinstance(eval_metric, metric.EvalMetric): eval_metric = metric.create(eval_metric) ################################################################################ # training loop ################################################################################ for epoch in range(begin_epoch, num_epoch): tic = time.time() eval_metric.reset() for nbatch, data_batch in enumerate(train_data): if monitor is not None: monitor.tic() self.forward_backward(data_batch) self.update() self.update_metric(eval_metric, data_batch.label) if monitor is not None: monitor.toc_print() if batch_end_callback is not None: batch_end_params = BatchEndParam(epoch=epoch, nbatch=nbatch, eval_metric=eval_metric, locals=locals()) for callback in _as_list(batch_end_callback): callback(batch_end_params) # one epoch of training is finished for name, val in eval_metric.get_name_value(): self.logger.info('Epoch[%d] Train-%s=%f', epoch, name, val) toc = time.time() self.logger.info('Epoch[%d] Time cost=%.3f', epoch, (toc-tic)) # sync aux params across devices arg_params, aux_params = self.get_params() self.set_params(arg_params, aux_params) if epoch_end_callback is not None: for callback in _as_list(epoch_end_callback): callback(epoch, self.symbol, arg_params, aux_params) #---------------------------------------- # evaluation on validation set if eval_data: res = self.score(eval_data, validation_metric, score_end_callback=eval_end_callback, batch_end_callback=eval_batch_end_callback, epoch=epoch) #TODO: pull this into default for name, val in res: self.logger.info('Epoch[%d] Validation-%s=%f', epoch, name, val) # end of 1 epoch, reset the data-iter for another epoch train_data.reset() def forward(self, data_batch, is_train=None): assert self.binded and self.params_initialized # get current_shapes if self._curr_module.label_shapes is not None: current_shapes = [dict(self._curr_module.data_shapes[i] + self._curr_module.label_shapes[i]) for i in xrange(len(self._context))] else: current_shapes = [dict(self._curr_module.data_shapes[i]) for i in xrange(len(self._context))] # get input_shapes if is_train: input_shapes = [dict(data_batch.provide_data[i] + data_batch.provide_label[i]) for i in xrange(len(self._context))] else: input_shapes = [dict(data_batch.provide_data[i]) for i in xrange(len(data_batch.provide_data))] # decide if shape changed shape_changed = len(current_shapes) != len(input_shapes) for pre, cur in zip(current_shapes, input_shapes): for k, v in pre.items(): if v != cur[k]: shape_changed = True if shape_changed: # self._curr_module.reshape(data_batch.provide_data, data_batch.provide_label) module = Module(self._symbol, self._data_names, self._label_names, logger=self.logger, context=[self._context[i] for i in xrange(len(data_batch.provide_data))], work_load_list=self._work_load_list, fixed_param_names=self._fixed_param_names) module.bind(data_batch.provide_data, data_batch.provide_label, self._curr_module.for_training, self._curr_module.inputs_need_grad, force_rebind=False, shared_module=self._curr_module) self._curr_module = module self._curr_module.forward(data_batch, is_train=is_train) def backward(self, out_grads=None): assert self.binded and self.params_initialized self._curr_module.backward(out_grads=out_grads) def update(self): assert self.binded and self.params_initialized and self.optimizer_initialized self._curr_module.update() def get_outputs(self, merge_multi_context=True): assert self.binded and self.params_initialized return self._curr_module.get_outputs(merge_multi_context=merge_multi_context) def get_input_grads(self, merge_multi_context=True): assert self.binded and self.params_initialized and self.inputs_need_grad return self._curr_module.get_input_grads(merge_multi_context=merge_multi_context) def update_metric(self, eval_metric, labels): assert self.binded and self.params_initialized self._curr_module.update_metric(eval_metric, labels) def install_monitor(self, mon): """ Install monitor on all executors """ assert self.binded self._curr_module.install_monitor(mon) ================================================ FILE: dff_rfcn/core/rcnn.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- """ Fast R-CNN: data = {'data': [num_images, c, h, w], 'rois': [num_rois, 5]} label = {'label': [num_rois], 'bbox_target': [num_rois, 4 * num_classes], 'bbox_weight': [num_rois, 4 * num_classes]} roidb extended format [image_index] ['image', 'height', 'width', 'flipped', 'boxes', 'gt_classes', 'gt_overlaps', 'max_classes', 'max_overlaps', 'bbox_targets'] """ import numpy as np import numpy.random as npr from utils.image import get_image, tensor_vstack from bbox.bbox_transform import bbox_overlaps, bbox_transform from bbox.bbox_regression import expand_bbox_regression_targets def get_rcnn_testbatch(roidb, cfg): """ return a dict of testbatch :param roidb: ['image', 'flipped'] + ['boxes'] :return: data, label, im_info """ # assert len(roidb) == 1, 'Single batch only' imgs, roidb = get_image(roidb, cfg) im_array = imgs im_info = [np.array([roidb[i]['im_info']], dtype=np.float32) for i in range(len(roidb))] im_rois = [roidb[i]['boxes'] for i in range(len(roidb))] rois = im_rois rois_array = [np.hstack((0 * np.ones((rois[i].shape[0], 1)), rois[i])) for i in range(len(rois))] data = [{'data': im_array[i], 'rois': rois_array[i]} for i in range(len(roidb))] label = {} return data, label, im_info def get_rcnn_batch(roidb, cfg): """ return a dict of multiple images :param roidb: a list of dict, whose length controls batch size ['images', 'flipped'] + ['gt_boxes', 'boxes', 'gt_overlap'] => ['bbox_targets'] :return: data, label """ num_images = len(roidb) imgs, roidb = get_image(roidb, cfg) im_array = tensor_vstack(imgs) assert cfg.TRAIN.BATCH_ROIS == -1 or cfg.TRAIN.BATCH_ROIS % cfg.TRAIN.BATCH_IMAGES == 0, \ 'BATCHIMAGES {} must divide BATCH_ROIS {}'.format(cfg.TRAIN.BATCH_IMAGES, cfg.TRAIN.BATCH_ROIS) if cfg.TRAIN.BATCH_ROIS == -1: rois_per_image = np.sum([iroidb['boxes'].shape[0] for iroidb in roidb]) fg_rois_per_image = rois_per_image else: rois_per_image = cfg.TRAIN.BATCH_ROIS / cfg.TRAIN.BATCH_IMAGES fg_rois_per_image = np.round(cfg.TRAIN.FG_FRACTION * rois_per_image).astype(int) rois_array = list() labels_array = list() bbox_targets_array = list() bbox_weights_array = list() for im_i in range(num_images): roi_rec = roidb[im_i] # infer num_classes from gt_overlaps num_classes = roi_rec['gt_overlaps'].shape[1] # label = class RoI has max overlap with rois = roi_rec['boxes'] labels = roi_rec['max_classes'] overlaps = roi_rec['max_overlaps'] bbox_targets = roi_rec['bbox_targets'] im_rois, labels, bbox_targets, bbox_weights = \ sample_rois(rois, fg_rois_per_image, rois_per_image, num_classes, cfg, labels, overlaps, bbox_targets) # project im_rois # do not round roi rois = im_rois batch_index = im_i * np.ones((rois.shape[0], 1)) rois_array_this_image = np.hstack((batch_index, rois)) rois_array.append(rois_array_this_image) # add labels labels_array.append(labels) bbox_targets_array.append(bbox_targets) bbox_weights_array.append(bbox_weights) rois_array = np.array(rois_array) labels_array = np.array(labels_array) bbox_targets_array = np.array(bbox_targets_array) bbox_weights_array = np.array(bbox_weights_array) data = {'data': im_array, 'rois': rois_array} label = {'label': labels_array, 'bbox_target': bbox_targets_array, 'bbox_weight': bbox_weights_array} return data, label def sample_rois(rois, fg_rois_per_image, rois_per_image, num_classes, cfg, labels=None, overlaps=None, bbox_targets=None, gt_boxes=None): """ generate random sample of ROIs comprising foreground and background examples :param rois: all_rois [n, 4]; e2e: [n, 5] with batch_index :param fg_rois_per_image: foreground roi number :param rois_per_image: total roi number :param num_classes: number of classes :param labels: maybe precomputed :param overlaps: maybe precomputed (max_overlaps) :param bbox_targets: maybe precomputed :param gt_boxes: optional for e2e [n, 5] (x1, y1, x2, y2, cls) :return: (labels, rois, bbox_targets, bbox_weights) """ if labels is None: overlaps = bbox_overlaps(rois[:, 1:].astype(np.float), gt_boxes[:, :4].astype(np.float)) gt_assignment = overlaps.argmax(axis=1) overlaps = overlaps.max(axis=1) labels = gt_boxes[gt_assignment, 4] # foreground RoI with FG_THRESH overlap fg_indexes = np.where(overlaps >= cfg.TRAIN.FG_THRESH)[0] # guard against the case when an image has fewer than fg_rois_per_image foreground RoIs fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_indexes.size) # Sample foreground regions without replacement if len(fg_indexes) > fg_rois_per_this_image: fg_indexes = npr.choice(fg_indexes, size=fg_rois_per_this_image, replace=False) # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_indexes = np.where((overlaps < cfg.TRAIN.BG_THRESH_HI) & (overlaps >= cfg.TRAIN.BG_THRESH_LO))[0] # Compute number of background RoIs to take from this image (guarding against there being fewer than desired) bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_indexes.size) # Sample foreground regions without replacement if len(bg_indexes) > bg_rois_per_this_image: bg_indexes = npr.choice(bg_indexes, size=bg_rois_per_this_image, replace=False) # indexes selected keep_indexes = np.append(fg_indexes, bg_indexes) # pad more to ensure a fixed minibatch size while keep_indexes.shape[0] < rois_per_image: gap = np.minimum(len(rois), rois_per_image - keep_indexes.shape[0]) gap_indexes = npr.choice(range(len(rois)), size=gap, replace=False) keep_indexes = np.append(keep_indexes, gap_indexes) # select labels labels = labels[keep_indexes] # set labels of bg_rois to be 0 labels[fg_rois_per_this_image:] = 0 rois = rois[keep_indexes] # load or compute bbox_target if bbox_targets is not None: bbox_target_data = bbox_targets[keep_indexes, :] else: targets = bbox_transform(rois[:, 1:], gt_boxes[gt_assignment[keep_indexes], :4]) if cfg.TRAIN.BBOX_NORMALIZATION_PRECOMPUTED: targets = ((targets - np.array(cfg.TRAIN.BBOX_MEANS)) / np.array(cfg.TRAIN.BBOX_STDS)) bbox_target_data = np.hstack((labels[:, np.newaxis], targets)) bbox_targets, bbox_weights = \ expand_bbox_regression_targets(bbox_target_data, num_classes, cfg) return rois, labels, bbox_targets, bbox_weights ================================================ FILE: dff_rfcn/core/tester.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Xizhou Zhu, Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- from multiprocessing.pool import ThreadPool as Pool import cPickle import os import time import mxnet as mx import numpy as np from module import MutableModule from utils import image from bbox.bbox_transform import bbox_pred, clip_boxes from nms.nms import py_nms_wrapper, cpu_nms_wrapper, gpu_nms_wrapper from utils.PrefetchingIter import PrefetchingIter class Predictor(object): def __init__(self, symbol, data_names, label_names, context=mx.cpu(), max_data_shapes=None, provide_data=None, provide_label=None, arg_params=None, aux_params=None): self._mod = MutableModule(symbol, data_names, label_names, context=context, max_data_shapes=max_data_shapes) self._mod.bind(provide_data, provide_label, for_training=False) self._mod.init_params(arg_params=arg_params, aux_params=aux_params) def predict(self, data_batch): self._mod.forward(data_batch) # [dict(zip(self._mod.output_names, _)) for _ in zip(*self._mod.get_outputs(merge_multi_context=False))] return [dict(zip(self._mod.output_names, _)) for _ in zip(*self._mod.get_outputs(merge_multi_context=False))] def im_proposal(predictor, data_batch, data_names, scales): output_all = predictor.predict(data_batch) data_dict_all = [dict(zip(data_names, data_batch.data[i])) for i in xrange(len(data_batch.data))] scores_all = [] boxes_all = [] for output, data_dict, scale in zip(output_all, data_dict_all, scales): # drop the batch index boxes = output['rois_output'].asnumpy()[:, 1:] scores = output['rois_score'].asnumpy() # transform to original scale boxes = boxes / scale scores_all.append(scores) boxes_all.append(boxes) return scores_all, boxes_all, data_dict_all def generate_proposals(predictor, test_data, imdb, cfg, vis=False, thresh=0.): """ Generate detections results using RPN. :param predictor: Predictor :param test_data: data iterator, must be non-shuffled :param imdb: image database :param vis: controls visualization :param thresh: thresh for valid detections :return: list of detected boxes """ assert vis or not test_data.shuffle data_names = [k[0] for k in test_data.provide_data[0]] if not isinstance(test_data, PrefetchingIter): test_data = PrefetchingIter(test_data) idx = 0 t = time.time() imdb_boxes = list() original_boxes = list() for im_info, data_batch in test_data: t1 = time.time() - t t = time.time() scales = [iim_info[0, 2] for iim_info in im_info] scores_all, boxes_all, data_dict_all = im_proposal(predictor, data_batch, data_names, scales) t2 = time.time() - t t = time.time() for delta, (scores, boxes, data_dict, scale) in enumerate(zip(scores_all, boxes_all, data_dict_all, scales)): # assemble proposals dets = np.hstack((boxes, scores)) original_boxes.append(dets) # filter proposals keep = np.where(dets[:, 4:] > thresh)[0] dets = dets[keep, :] imdb_boxes.append(dets) if vis: vis_all_detection(data_dict['data'].asnumpy(), [dets], ['obj'], scale, cfg) print 'generating %d/%d' % (idx + 1, imdb.num_images), 'proposal %d' % (dets.shape[0]), \ 'data %.4fs net %.4fs' % (t1, t2 / test_data.batch_size) idx += 1 assert len(imdb_boxes) == imdb.num_images, 'calculations not complete' # save results rpn_folder = os.path.join(imdb.result_path, 'rpn_data') if not os.path.exists(rpn_folder): os.mkdir(rpn_folder) rpn_file = os.path.join(rpn_folder, imdb.name + '_rpn.pkl') with open(rpn_file, 'wb') as f: cPickle.dump(imdb_boxes, f, cPickle.HIGHEST_PROTOCOL) if thresh > 0: full_rpn_file = os.path.join(rpn_folder, imdb.name + '_full_rpn.pkl') with open(full_rpn_file, 'wb') as f: cPickle.dump(original_boxes, f, cPickle.HIGHEST_PROTOCOL) print 'wrote rpn proposals to {}'.format(rpn_file) return imdb_boxes def im_detect(predictor, data_batch, data_names, scales, cfg): output_all = predictor.predict(data_batch) data_dict_all = [dict(zip(data_names, data_batch.data[i])) for i in xrange(len(data_batch.data))] scores_all = [] pred_boxes_all = [] for output, data_dict, scale in zip(output_all, data_dict_all, scales): if cfg.TEST.HAS_RPN: rois = output['rois_output'].asnumpy()[:, 1:] else: rois = data_dict['rois'].asnumpy().reshape((-1, 5))[:, 1:] im_shape = data_dict['data'].shape # save output scores = output['cls_prob_reshape_output'].asnumpy()[0] bbox_deltas = output['bbox_pred_reshape_output'].asnumpy()[0] # post processing pred_boxes = bbox_pred(rois, bbox_deltas) pred_boxes = clip_boxes(pred_boxes, im_shape[-2:]) # we used scaled image & roi to train, so it is necessary to transform them back pred_boxes = pred_boxes / scale scores_all.append(scores) pred_boxes_all.append(pred_boxes) if output_all[0].has_key('feat_conv_3x3_relu_output'): feat = output_all[0]['feat_conv_3x3_relu_output'] else: feat = None return scores_all, pred_boxes_all, data_dict_all, feat def im_batch_detect(predictor, data_batch, data_names, scales, cfg): output_all = predictor.predict(data_batch) data_dict_all = [dict(zip(data_names, data_batch.data[i])) for i in xrange(len(data_batch.data))] scores_all = [] pred_boxes_all = [] for output, data_dict, scale in zip(output_all, data_dict_all, scales): im_infos = data_dict['im_info'].asnumpy() # save output scores = output['cls_prob_reshape_output'].asnumpy()[0] bbox_deltas = output['bbox_pred_reshape_output'].asnumpy()[0] rois = output['rois_output'].asnumpy() for im_idx in xrange(im_infos.shape[0]): bb_idxs = np.where(rois[:,0] == im_idx)[0] im_shape = im_infos[im_idx, :2].astype(np.int) # post processing pred_boxes = bbox_pred(rois[bb_idxs, 1:], bbox_deltas[bb_idxs, :]) pred_boxes = clip_boxes(pred_boxes, im_shape) # we used scaled image & roi to train, so it is necessary to transform them back pred_boxes = pred_boxes / scale[im_idx] scores_all.append(scores[bb_idxs, :]) pred_boxes_all.append(pred_boxes) return scores_all, pred_boxes_all, data_dict_all def pred_eval(gpu_id, key_predictor, cur_predictor, test_data, imdb, cfg, vis=False, thresh=1e-4, logger=None, ignore_cache=True): """ wrapper for calculating offline validation for faster data analysis in this example, all threshold are set by hand :param predictor: Predictor :param test_data: data iterator, must be non-shuffle :param imdb: image database :param vis: controls visualization :param thresh: valid detection threshold :return: """ det_file = os.path.join(imdb.result_path, imdb.name + '_'+ str(gpu_id) + '_detections.pkl') if os.path.exists(det_file) and not ignore_cache: with open(det_file, 'rb') as fid: all_boxes, frame_ids = cPickle.load(fid) return all_boxes, frame_ids assert vis or not test_data.shuffle data_names = [k[0] for k in test_data.provide_data[0]] num_images = test_data.size roidb_frame_ids = [x['frame_id'] for x in test_data.roidb] if not isinstance(test_data, PrefetchingIter): test_data = PrefetchingIter(test_data) nms = py_nms_wrapper(cfg.TEST.NMS) # limit detections to max_per_image over all classes max_per_image = cfg.TEST.max_per_image # all detections are collected into: # all_boxes[cls][image] = N x 5 array of detections in # (x1, y1, x2, y2, score) all_boxes = [[[] for _ in range(num_images)] for _ in range(imdb.num_classes)] frame_ids = np.zeros(num_images, dtype=np.int) roidb_idx = -1 roidb_offset = -1 idx = 0 data_time, net_time, post_time = 0.0, 0.0, 0.0 t = time.time() for im_info, key_frame_flag, data_batch in test_data: t1 = time.time() - t t = time.time() scales = [iim_info[0, 2] for iim_info in im_info] if key_frame_flag != 2: scores_all, boxes_all, data_dict_all, feat = im_detect(key_predictor, data_batch, data_names, scales, cfg) else: data_batch.data[0][-1] = feat data_batch.provide_data[0][-1] = ('feat_key', feat.shape) scores_all, boxes_all, data_dict_all, _ = im_detect(cur_predictor, data_batch, data_names, scales, cfg) if key_frame_flag == 0: roidb_idx += 1 roidb_offset = 0 else: roidb_offset += 1 frame_ids[idx] = roidb_frame_ids[roidb_idx] + roidb_offset t2 = time.time() - t t = time.time() for delta, (scores, boxes, data_dict) in enumerate(zip(scores_all, boxes_all, data_dict_all)): for j in range(1, imdb.num_classes): indexes = np.where(scores[:, j] > thresh)[0] cls_scores = scores[indexes, j, np.newaxis] cls_boxes = boxes[indexes, 4:8] if cfg.CLASS_AGNOSTIC else boxes[indexes, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) all_boxes[j][idx+delta] = cls_dets[keep, :] if max_per_image > 0: image_scores = np.hstack([all_boxes[j][idx+delta][:, -1] for j in range(1, imdb.num_classes)]) if len(image_scores) > max_per_image: image_thresh = np.sort(image_scores)[-max_per_image] for j in range(1, imdb.num_classes): keep = np.where(all_boxes[j][idx+delta][:, -1] >= image_thresh)[0] all_boxes[j][idx+delta] = all_boxes[j][idx+delta][keep, :] if vis: boxes_this_image = [[]] + [all_boxes[j][idx+delta] for j in range(1, imdb.num_classes)] vis_all_detection(data_dict['data'].asnumpy(), boxes_this_image, imdb.classes, scales[delta], cfg) idx += test_data.batch_size t3 = time.time() - t t = time.time() data_time += t1 net_time += t2 post_time += t3 print 'testing {}/{} data {:.4f}s net {:.4f}s post {:.4f}s'.format(idx, num_images, data_time / idx * test_data.batch_size, net_time / idx * test_data.batch_size, post_time / idx * test_data.batch_size) if logger: logger.info('testing {}/{} data {:.4f}s net {:.4f}s post {:.4f}s'.format(idx, num_images, data_time / idx * test_data.batch_size, net_time / idx * test_data.batch_size, post_time / idx * test_data.batch_size)) with open(det_file, 'wb') as f: cPickle.dump((all_boxes, frame_ids), f, protocol=cPickle.HIGHEST_PROTOCOL) return all_boxes, frame_ids def pred_eval_multiprocess(gpu_num, key_predictors, cur_predictors, test_datas, imdb, cfg, vis=False, thresh=1e-4, logger=None, ignore_cache=True): if gpu_num == 1: res = [pred_eval(0, key_predictors[0], cur_predictors[0], test_datas[0], imdb, cfg, vis, thresh, logger, ignore_cache),] else: pool = Pool(processes=gpu_num) multiple_results = [pool.apply_async(pred_eval,args=(i, key_predictors[i], cur_predictors[i], test_datas[i], imdb, cfg, vis, thresh, logger, ignore_cache)) for i in range(gpu_num)] pool.close() pool.join() res = [res.get() for res in multiple_results] info_str = imdb.evaluate_detections_multiprocess(res) if logger: logger.info('evaluate detections: \n{}'.format(info_str)) def vis_all_detection(im_array, detections, class_names, scale, cfg, threshold=1e-4): """ visualize all detections in one image :param im_array: [b=1 c h w] in rgb :param detections: [ numpy.ndarray([[x1 y1 x2 y2 score]]) for j in classes ] :param class_names: list of names in imdb :param scale: visualize the scaled image :return: """ import matplotlib.pyplot as plt import random im = image.transform_inverse(im_array, cfg.network.PIXEL_MEANS) plt.imshow(im) for j, name in enumerate(class_names): if name == '__background__': continue color = (random.random(), random.random(), random.random()) # generate a random color dets = detections[j] for det in dets: bbox = det[:4] * scale score = det[-1] if score < threshold: continue rect = plt.Rectangle((bbox[0], bbox[1]), bbox[2] - bbox[0], bbox[3] - bbox[1], fill=False, edgecolor=color, linewidth=3.5) plt.gca().add_patch(rect) plt.gca().text(bbox[0], bbox[1] - 2, '{:s} {:.3f}'.format(name, score), bbox=dict(facecolor=color, alpha=0.5), fontsize=12, color='white') plt.show() def draw_all_detection(im_array, detections, class_names, scale, cfg, threshold=1e-1): """ visualize all detections in one image :param im_array: [b=1 c h w] in rgb :param detections: [ numpy.ndarray([[x1 y1 x2 y2 score]]) for j in classes ] :param class_names: list of names in imdb :param scale: visualize the scaled image :return: """ import cv2 import random color_white = (255, 255, 255) im = image.transform_inverse(im_array, cfg.network.PIXEL_MEANS) # change to bgr im = cv2.cvtColor(im, cv2.COLOR_RGB2BGR) for j, name in enumerate(class_names): if name == '__background__': continue color = (random.randint(0, 256), random.randint(0, 256), random.randint(0, 256)) # generate a random color dets = detections[j] for det in dets: bbox = det[:4] * scale score = det[-1] if score < threshold: continue bbox = map(int, bbox) cv2.rectangle(im, (bbox[0], bbox[1]), (bbox[2], bbox[3]), color=color, thickness=2) cv2.putText(im, '%s %.3f' % (class_names[j], score), (bbox[0], bbox[1] + 10), color=color_white, fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.5) return im ================================================ FILE: dff_rfcn/demo.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Xizhou Zhu, Yi Li, Haochen Zhang # -------------------------------------------------------- import _init_paths import argparse import os import glob import sys import logging import pprint import cv2 from config.config import config, update_config from utils.image import resize, transform import numpy as np # get config os.environ['PYTHONUNBUFFERED'] = '1' os.environ['MXNET_CUDNN_AUTOTUNE_DEFAULT'] = '0' os.environ['MXNET_ENABLE_GPU_P2P'] = '0' cur_path = os.path.abspath(os.path.dirname(__file__)) update_config(cur_path + '/../experiments/dff_rfcn/cfgs/dff_rfcn_vid_demo.yaml') sys.path.insert(0, os.path.join(cur_path, '../external/mxnet', config.MXNET_VERSION)) import mxnet as mx from core.tester import im_detect, Predictor from symbols import * from utils.load_model import load_param from utils.show_boxes import show_boxes, draw_boxes from utils.tictoc import tic, toc from nms.nms import py_nms_wrapper, cpu_nms_wrapper, gpu_nms_wrapper def parse_args(): parser = argparse.ArgumentParser(description='Show Deep Feature Flow demo') args = parser.parse_args() return args args = parse_args() def main(): # get symbol pprint.pprint(config) config.symbol = 'resnet_v1_101_flownet_rfcn' model = '/../model/rfcn_dff_flownet_vid' sym_instance = eval(config.symbol + '.' + config.symbol)() key_sym = sym_instance.get_key_test_symbol(config) cur_sym = sym_instance.get_cur_test_symbol(config) # set up class names num_classes = 31 classes = ['airplane', 'antelope', 'bear', 'bicycle', 'bird', 'bus', 'car', 'cattle', 'dog', 'domestic_cat', 'elephant', 'fox', 'giant_panda', 'hamster', 'horse', 'lion', 'lizard', 'monkey', 'motorcycle', 'rabbit', 'red_panda', 'sheep', 'snake', 'squirrel', 'tiger', 'train', 'turtle', 'watercraft', 'whale', 'zebra'] # load demo data image_names = glob.glob(cur_path + '/../demo/ILSVRC2015_val_00007010/*.JPEG') output_dir = cur_path + '/../demo/rfcn_dff/' if not os.path.exists(output_dir): os.makedirs(output_dir) key_frame_interval = 10 # data = [] key_im_tensor = None for idx, im_name in enumerate(image_names): assert os.path.exists(im_name), ('%s does not exist'.format(im_name)) im = cv2.imread(im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) if idx % key_frame_interval == 0: key_im_tensor = im_tensor data.append({'data': im_tensor, 'im_info': im_info, 'data_key': key_im_tensor, 'feat_key': np.zeros((1,config.network.DFF_FEAT_DIM,1,1))}) # get predictor data_names = ['data', 'im_info', 'data_key', 'feat_key'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES]))), ('data_key', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES]))),]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] arg_params, aux_params = load_param(cur_path + model, 0, process=True) key_predictor = Predictor(key_sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) cur_predictor = Predictor(cur_sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) nms = gpu_nms_wrapper(config.TEST.NMS, 0) # warm up for j in xrange(2): data_batch = mx.io.DataBatch(data=[data[j]], label=[], pad=0, index=0, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[j])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))] if j % key_frame_interval == 0: scores, boxes, data_dict, feat = im_detect(key_predictor, data_batch, data_names, scales, config) else: data_batch.data[0][-1] = feat data_batch.provide_data[0][-1] = ('feat_key', feat.shape) scores, boxes, data_dict, _ = im_detect(cur_predictor, data_batch, data_names, scales, config) print "warmup done" # test time = 0 count = 0 for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))] tic() if idx % key_frame_interval == 0: scores, boxes, data_dict, feat = im_detect(key_predictor, data_batch, data_names, scales, config) else: data_batch.data[0][-1] = feat data_batch.provide_data[0][-1] = ('feat_key', feat.shape) scores, boxes, data_dict, _ = im_detect(cur_predictor, data_batch, data_names, scales, config) time += toc() count += 1 print 'testing {} {:.4f}s'.format(im_name, time/count) boxes = boxes[0].astype('f') scores = scores[0].astype('f') dets_nms = [] for j in range(1, scores.shape[1]): cls_scores = scores[:, j, np.newaxis] cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) cls_dets = cls_dets[keep, :] cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :] dets_nms.append(cls_dets) # visualize im = cv2.imread(im_name) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) # show_boxes(im, dets_nms, classes, 1) out_im = draw_boxes(im, dets_nms, classes, 1) _, filename = os.path.split(im_name) cv2.imwrite(output_dir + filename,out_im) print 'done' if __name__ == '__main__': main() ================================================ FILE: dff_rfcn/demo_batch.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Xizhou Zhu, Yi Li, Haochen Zhang # -------------------------------------------------------- import _init_paths import argparse import os import glob import sys import logging import pprint import cv2 from config.config import config, update_config from utils.image import resize, transform import numpy as np # get config os.environ['PYTHONUNBUFFERED'] = '1' os.environ['MXNET_CUDNN_AUTOTUNE_DEFAULT'] = '0' os.environ['MXNET_ENABLE_GPU_P2P'] = '0' cur_path = os.path.abspath(os.path.dirname(__file__)) update_config(cur_path + '/../experiments/dff_rfcn/cfgs/dff_rfcn_vid_demo.yaml') sys.path.insert(0, os.path.join(cur_path, '../external/mxnet', config.MXNET_VERSION)) import mxnet as mx from core.tester import im_batch_detect, Predictor from symbols import * from utils.load_model import load_param from utils.show_boxes import show_boxes, draw_boxes from utils.tictoc import tic, toc from nms.nms import py_nms_wrapper, cpu_nms_wrapper, gpu_nms_wrapper def parse_args(): parser = argparse.ArgumentParser(description='Show Deep Feature Flow demo') args = parser.parse_args() return args args = parse_args() def main(): # get symbol pprint.pprint(config) config.symbol = 'resnet_v1_101_flownet_rfcn' model = '/../model/rfcn_dff_flownet_vid' sym_instance = eval(config.symbol + '.' + config.symbol)() sym = sym_instance.get_batch_test_symbol(config) # set up class names num_classes = 31 classes = ['airplane', 'antelope', 'bear', 'bicycle', 'bird', 'bus', 'car', 'cattle', 'dog', 'domestic_cat', 'elephant', 'fox', 'giant_panda', 'hamster', 'horse', 'lion', 'lizard', 'monkey', 'motorcycle', 'rabbit', 'red_panda', 'sheep', 'snake', 'squirrel', 'tiger', 'train', 'turtle', 'watercraft', 'whale', 'zebra'] # load demo data image_names = glob.glob(cur_path + '/../demo/ILSVRC2015_val_00007010/*.JPEG') output_dir = cur_path + '/../demo/rfcn_dff_batch/' if not os.path.exists(output_dir): os.makedirs(output_dir) key_frame_interval = 10 # data = [] key_im_tensor = None cur_im_tensor = [] im_info_tensor = [] image_names_list = [] image_names_batch = [] for idx, im_name in enumerate(image_names): assert os.path.exists(im_name), ('%s does not exist'.format(im_name)) im = cv2.imread(im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) if idx % key_frame_interval == 0: key_im_tensor = im_tensor else: cur_im_tensor.append(im_tensor) im_info_tensor.append(im_info) image_names_batch.append(im_name) if (idx+1) % key_frame_interval == 0 or idx == len(image_names) - 1: data.append({'data_other': np.concatenate(cur_im_tensor), 'im_info': np.concatenate(im_info_tensor), 'data_key': key_im_tensor}) key_im_tensor = None cur_im_tensor = [] im_info_tensor = [] image_names_list.append(image_names_batch) image_names_batch = [] # get predictor data_names = ['data_other', 'im_info', 'data_key'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data_other', (key_frame_interval-1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES]))), ('data_key', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES]))),]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] arg_params, aux_params = load_param(cur_path + model, 0, process=True) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) nms = gpu_nms_wrapper(config.TEST.NMS, 0) # warm up for j in xrange(1): data_batch = mx.io.DataBatch(data=[data[j]], label=[], pad=0, index=0, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[j])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[:, 2] for i in xrange(len(data_batch.data))] scores_all, boxes_all, data_dict = im_batch_detect(predictor, data_batch, data_names, scales, config) print "warmup done" # test time = 0 count = 0 for idx, im_names in enumerate(image_names_list): data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[:, 2] for i in xrange(len(data_batch.data))] tic() scores_all, boxes_all, data_dict = im_batch_detect(predictor, data_batch, data_names, scales, config) time += toc() count += len(scores_all) print 'testing {} {:.4f}s x {:d}'.format(im_names[0], time/count, len(scores_all)) for batch_idx in xrange(len(scores_all)): boxes = boxes_all[batch_idx].astype('f') scores = scores_all[batch_idx].astype('f') dets_nms = [] for j in range(1, scores.shape[1]): cls_scores = scores[:, j, np.newaxis] cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) cls_dets = cls_dets[keep, :] cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :] dets_nms.append(cls_dets) # visualize im = cv2.imread(im_names[batch_idx]) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) # show_boxes(im, dets_nms, classes, 1) out_im = draw_boxes(im, dets_nms, classes, 1) _, filename = os.path.split(im_names[batch_idx]) cv2.imwrite(output_dir + filename,out_im) print 'done' if __name__ == '__main__': main() ================================================ FILE: dff_rfcn/function/__init__.py ================================================ ================================================ FILE: dff_rfcn/function/test_rcnn.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- import argparse import pprint import logging import time import os import numpy as np import mxnet as mx from symbols import * from dataset import * from core.loader import TestLoader from core.tester import Predictor, pred_eval, pred_eval_multiprocess from utils.load_model import load_param def get_predictor(sym, sym_instance, cfg, arg_params, aux_params, test_data, ctx): # infer shape data_shape_dict = dict(test_data.provide_data_single) sym_instance.infer_shape(data_shape_dict) sym_instance.check_parameter_shapes(arg_params, aux_params, data_shape_dict, is_train=False) # decide maximum shape data_names = [k[0] for k in test_data.provide_data_single] label_names = None max_data_shape = [[('data', (1, 3, max([v[0] for v in cfg.SCALES]), max([v[1] for v in cfg.SCALES]))), ('data_key', (1, 3, max([v[0] for v in cfg.SCALES]), max([v[1] for v in cfg.SCALES]))),]] # create predictor predictor = Predictor(sym, data_names, label_names, context=ctx, max_data_shapes=max_data_shape, provide_data=test_data.provide_data, provide_label=test_data.provide_label, arg_params=arg_params, aux_params=aux_params) return predictor def test_rcnn(cfg, dataset, image_set, root_path, dataset_path, ctx, prefix, epoch, vis, ignore_cache, shuffle, has_rpn, proposal, thresh, logger=None, output_path=None): if not logger: assert False, 'require a logger' # print cfg pprint.pprint(cfg) logger.info('testing cfg:{}\n'.format(pprint.pformat(cfg))) # load symbol and testing data key_sym_instance = eval(cfg.symbol + '.' + cfg.symbol)() cur_sym_instance = eval(cfg.symbol + '.' + cfg.symbol)() key_sym = key_sym_instance.get_key_test_symbol(cfg) cur_sym = cur_sym_instance.get_cur_test_symbol(cfg) imdb = eval(dataset)(image_set, root_path, dataset_path, result_path=output_path) roidb = imdb.gt_roidb() # get test data iter # split roidbs gpu_num = len(ctx) roidbs = [[] for x in range(gpu_num)] roidbs_seg_lens = np.zeros(gpu_num, dtype=np.int) for x in roidb: gpu_id = np.argmin(roidbs_seg_lens) roidbs[gpu_id].append(x) roidbs_seg_lens[gpu_id] += x['frame_seg_len'] # get test data iter test_datas = [TestLoader(x, cfg, batch_size=1, shuffle=shuffle, has_rpn=has_rpn) for x in roidbs] # load model arg_params, aux_params = load_param(prefix, epoch, process=True) # create predictor key_predictors = [get_predictor(key_sym, key_sym_instance, cfg, arg_params, aux_params, test_datas[i], [ctx[i]]) for i in range(gpu_num)] cur_predictors = [get_predictor(cur_sym, cur_sym_instance, cfg, arg_params, aux_params, test_datas[i], [ctx[i]]) for i in range(gpu_num)] # start detection #pred_eval(0, key_predictors[0], cur_predictors[0], test_datas[0], imdb, cfg, vis=vis, ignore_cache=ignore_cache, thresh=thresh, logger=logger) pred_eval_multiprocess(gpu_num, key_predictors, cur_predictors, test_datas, imdb, cfg, vis=vis, ignore_cache=ignore_cache, thresh=thresh, logger=logger) ================================================ FILE: dff_rfcn/function/test_rpn.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- import argparse import pprint import logging import mxnet as mx from symbols import * from dataset import * from core.loader import TestLoader from core.tester import Predictor, generate_proposals from utils.load_model import load_param def test_rpn(cfg, dataset, image_set, root_path, dataset_path, ctx, prefix, epoch, vis, shuffle, thresh, logger=None, output_path=None): # set up logger if not logger: logging.basicConfig() logger = logging.getLogger() logger.setLevel(logging.INFO) # rpn generate proposal cfg cfg.TEST.HAS_RPN = True # print cfg pprint.pprint(cfg) logger.info('testing rpn cfg:{}\n'.format(pprint.pformat(cfg))) # load symbol sym_instance = eval(cfg.symbol + '.' + cfg.symbol)() sym = sym_instance.get_symbol_rpn(cfg, is_train=False) # load dataset and prepare imdb for training imdb = eval(dataset)(image_set, root_path, dataset_path, result_path=output_path) roidb = imdb.gt_roidb() test_data = TestLoader(roidb, cfg, batch_size=len(ctx), shuffle=shuffle, has_rpn=True) # load model arg_params, aux_params = load_param(prefix, epoch) # infer shape data_shape_dict = dict(test_data.provide_data_single) sym_instance.infer_shape(data_shape_dict) # check parameters sym_instance.check_parameter_shapes(arg_params, aux_params, data_shape_dict, is_train=False) # decide maximum shape data_names = [k[0] for k in test_data.provide_data[0]] label_names = None if test_data.provide_label[0] is None else [k[0] for k in test_data.provide_label[0]] max_data_shape = [[('data', (1, 3, max([v[0] for v in cfg.SCALES]), max([v[1] for v in cfg.SCALES])))]] # create predictor predictor = Predictor(sym, data_names, label_names, context=ctx, max_data_shapes=max_data_shape, provide_data=test_data.provide_data, provide_label=test_data.provide_label, arg_params=arg_params, aux_params=aux_params) # start testing imdb_boxes = generate_proposals(predictor, test_data, imdb, cfg, vis=vis, thresh=thresh) all_log_info = imdb.evaluate_recall(roidb, candidate_boxes=imdb_boxes) logger.info(all_log_info) ================================================ FILE: dff_rfcn/function/train_rcnn.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- import argparse import logging import pprint import os import mxnet as mx from symbols import * from core import callback, metric from core.loader import ROIIter from core.module import MutableModule from bbox.bbox_regression import add_bbox_regression_targets from utils.load_data import load_proposal_roidb, merge_roidb, filter_roidb from utils.load_model import load_param from utils.PrefetchingIter import PrefetchingIter from utils.lr_scheduler import WarmupMultiFactorScheduler def train_rcnn(cfg, dataset, image_set, root_path, dataset_path, frequent, kvstore, flip, shuffle, resume, ctx, pretrained, epoch, prefix, begin_epoch, end_epoch, train_shared, lr, lr_step, proposal, logger=None, output_path=None): # set up logger if not logger: logging.basicConfig() logger = logging.getLogger() logger.setLevel(logging.INFO) # load symbol sym_instance = eval(cfg.symbol + '.' + cfg.symbol)() sym = sym_instance.get_symbol_rfcn(cfg, is_train=True) # setup multi-gpu batch_size = len(ctx) input_batch_size = cfg.TRAIN.BATCH_IMAGES * batch_size # print cfg pprint.pprint(cfg) logger.info('training rcnn cfg:{}\n'.format(pprint.pformat(cfg))) # load dataset and prepare imdb for training image_sets = [iset for iset in image_set.split('+')] roidbs = [load_proposal_roidb(dataset, image_set, root_path, dataset_path, proposal=proposal, append_gt=True, flip=flip, result_path=output_path) for image_set in image_sets] roidb = merge_roidb(roidbs) roidb = filter_roidb(roidb, cfg) means, stds = add_bbox_regression_targets(roidb, cfg) # load training data train_data = ROIIter(roidb, cfg, batch_size=input_batch_size, shuffle=shuffle, ctx=ctx, aspect_grouping=cfg.TRAIN.ASPECT_GROUPING) # infer max shape max_data_shape = [('data', (cfg.TRAIN.BATCH_IMAGES, 3, max([v[0] for v in cfg.SCALES]), max([v[1] for v in cfg.SCALES])))] # infer shape data_shape_dict = dict(train_data.provide_data_single + train_data.provide_label_single) sym_instance.infer_shape(data_shape_dict) # load and initialize params if resume: print('continue training from ', begin_epoch) arg_params, aux_params = load_param(prefix, begin_epoch, convert=True) else: arg_params, aux_params = load_param(pretrained, epoch, convert=True) sym_instance.init_weight_rfcn(cfg, arg_params, aux_params) # check parameter shapes sym_instance.check_parameter_shapes(arg_params, aux_params, data_shape_dict) # prepare training # create solver data_names = [k[0] for k in train_data.provide_data_single] label_names = [k[0] for k in train_data.provide_label_single] if train_shared: fixed_param_prefix = cfg.network.FIXED_PARAMS_SHARED else: fixed_param_prefix = cfg.network.FIXED_PARAMS mod = MutableModule(sym, data_names=data_names, label_names=label_names, logger=logger, context=ctx, max_data_shapes=[max_data_shape for _ in range(batch_size)], fixed_param_prefix=fixed_param_prefix) if cfg.TRAIN.RESUME: mod._preload_opt_states = '%s-%04d.states'%(prefix, begin_epoch) # decide training params # metric eval_metric = metric.RCNNAccMetric(cfg) cls_metric = metric.RCNNLogLossMetric(cfg) bbox_metric = metric.RCNNL1LossMetric(cfg) eval_metrics = mx.metric.CompositeEvalMetric() for child_metric in [eval_metric, cls_metric, bbox_metric]: eval_metrics.add(child_metric) # callback batch_end_callback = callback.Speedometer(train_data.batch_size, frequent=frequent) epoch_end_callback = [mx.callback.module_checkpoint(mod, prefix, period=1, save_optimizer_states=True), callback.do_checkpoint(prefix, means, stds)] # decide learning rate base_lr = lr lr_factor = cfg.TRAIN.lr_factor lr_epoch = [float(epoch) for epoch in lr_step.split(',')] lr_epoch_diff = [epoch - begin_epoch for epoch in lr_epoch if epoch > begin_epoch] lr = base_lr * (lr_factor ** (len(lr_epoch) - len(lr_epoch_diff))) lr_iters = [int(epoch * len(roidb) / batch_size) for epoch in lr_epoch_diff] print('lr', lr, 'lr_epoch_diff', lr_epoch_diff, 'lr_iters', lr_iters) lr_scheduler = WarmupMultiFactorScheduler(lr_iters, lr_factor, cfg.TRAIN.warmup, cfg.TRAIN.warmup_lr, cfg.TRAIN.warmup_step) # optimizer optimizer_params = {'momentum': cfg.TRAIN.momentum, 'wd': cfg.TRAIN.wd, 'learning_rate': lr, 'lr_scheduler': lr_scheduler, 'rescale_grad': 1.0, 'clip_gradient': None} # train if not isinstance(train_data, PrefetchingIter): train_data = PrefetchingIter(train_data) mod.fit(train_data, eval_metric=eval_metrics, epoch_end_callback=epoch_end_callback, batch_end_callback=batch_end_callback, kvstore=kvstore, optimizer='sgd', optimizer_params=optimizer_params, arg_params=arg_params, aux_params=aux_params, begin_epoch=begin_epoch, num_epoch=end_epoch) ================================================ FILE: dff_rfcn/function/train_rpn.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- import argparse import logging import pprint import mxnet as mx from symbols import * from core import callback, metric from core.loader import AnchorLoader from core.module import MutableModule from utils.load_data import load_gt_roidb, merge_roidb, filter_roidb from utils.load_model import load_param from utils.PrefetchingIter import PrefetchingIter from utils.lr_scheduler import WarmupMultiFactorScheduler def train_rpn(cfg, dataset, image_set, root_path, dataset_path, frequent, kvstore, flip, shuffle, resume, ctx, pretrained, epoch, prefix, begin_epoch, end_epoch, train_shared, lr, lr_step, logger=None, output_path=None): # set up logger if not logger: logging.basicConfig() logger = logging.getLogger() logger.setLevel(logging.INFO) # set up config cfg.TRAIN.BATCH_IMAGES = cfg.TRAIN.ALTERNATE.RPN_BATCH_IMAGES # load symbol sym_instance = eval(cfg.symbol + '.' + cfg.symbol)() sym = sym_instance.get_symbol_rpn(cfg, is_train=True) feat_sym = sym.get_internals()['rpn_cls_score_output'] # setup multi-gpu batch_size = len(ctx) input_batch_size = cfg.TRAIN.BATCH_IMAGES * batch_size # print cfg pprint.pprint(cfg) logger.info('training rpn cfg:{}\n'.format(pprint.pformat(cfg))) # load dataset and prepare imdb for training image_sets = [iset for iset in image_set.split('+')] roidbs = [load_gt_roidb(dataset, image_set, root_path, dataset_path, result_path=output_path, flip=flip) for image_set in image_sets] roidb = merge_roidb(roidbs) roidb = filter_roidb(roidb, cfg) # load training data train_data = AnchorLoader(feat_sym, roidb, cfg, batch_size=input_batch_size, shuffle=shuffle, ctx=ctx, feat_stride=cfg.network.RPN_FEAT_STRIDE, anchor_scales=cfg.network.ANCHOR_SCALES, anchor_ratios=cfg.network.ANCHOR_RATIOS, aspect_grouping=cfg.TRAIN.ASPECT_GROUPING) # infer max shape max_data_shape = [('data', (cfg.TRAIN.BATCH_IMAGES, 3, max([v[0] for v in cfg.SCALES]), max([v[1] for v in cfg.SCALES])))] max_data_shape, max_label_shape = train_data.infer_shape(max_data_shape) print('providing maximum shape', max_data_shape, max_label_shape) # infer shape data_shape_dict = dict(train_data.provide_data_single + train_data.provide_label_single) sym_instance.infer_shape(data_shape_dict) # load and initialize params if resume: print('continue training from ', begin_epoch) arg_params, aux_params = load_param(prefix, begin_epoch, convert=True) else: arg_params, aux_params = load_param(pretrained, epoch, convert=True) sym_instance.init_weight_rpn(cfg, arg_params, aux_params) # check parameter shapes sym_instance.check_parameter_shapes(arg_params, aux_params, data_shape_dict) # create solver data_names = [k[0] for k in train_data.provide_data_single] label_names = [k[0] for k in train_data.provide_label_single] if train_shared: fixed_param_prefix = cfg.network.FIXED_PARAMS_SHARED else: fixed_param_prefix = cfg.network.FIXED_PARAMS mod = MutableModule(sym, data_names=data_names, label_names=label_names, logger=logger, context=ctx, max_data_shapes=[max_data_shape for _ in xrange(batch_size)], max_label_shapes=[max_label_shape for _ in xrange(batch_size)], fixed_param_prefix=fixed_param_prefix) # decide training params # metric eval_metric = metric.RPNAccMetric() cls_metric = metric.RPNLogLossMetric() bbox_metric = metric.RPNL1LossMetric() eval_metrics = mx.metric.CompositeEvalMetric() for child_metric in [eval_metric, cls_metric, bbox_metric]: eval_metrics.add(child_metric) # callback batch_end_callback = callback.Speedometer(train_data.batch_size, frequent=frequent) # epoch_end_callback = mx.callback.do_checkpoint(prefix) epoch_end_callback = mx.callback.module_checkpoint(mod, prefix, period=1, save_optimizer_states=True) # decide learning rate base_lr = lr lr_factor = cfg.TRAIN.lr_factor lr_epoch = [int(epoch) for epoch in lr_step.split(',')] lr_epoch_diff = [epoch - begin_epoch for epoch in lr_epoch if epoch > begin_epoch] lr = base_lr * (lr_factor ** (len(lr_epoch) - len(lr_epoch_diff))) lr_iters = [int(epoch * len(roidb) / batch_size) for epoch in lr_epoch_diff] print('lr', lr, 'lr_epoch_diff', lr_epoch_diff, 'lr_iters', lr_iters) lr_scheduler = WarmupMultiFactorScheduler(lr_iters, lr_factor, cfg.TRAIN.warmup, cfg.TRAIN.warmup_lr, cfg.TRAIN.warmup_step) # optimizer optimizer_params = {'momentum': cfg.TRAIN.momentum, 'wd': cfg.TRAIN.wd, 'learning_rate': lr, 'lr_scheduler': lr_scheduler, 'rescale_grad': 1.0, 'clip_gradient': None} if not isinstance(train_data, PrefetchingIter): train_data = PrefetchingIter(train_data) # train mod.fit(train_data, eval_metric=eval_metrics, epoch_end_callback=epoch_end_callback, batch_end_callback=batch_end_callback, kvstore=kvstore, optimizer='sgd', optimizer_params=optimizer_params, arg_params=arg_params, aux_params=aux_params, begin_epoch=begin_epoch, num_epoch=end_epoch) ================================================ FILE: dff_rfcn/operator_cxx/multi_proposal-inl.h ================================================ /*! * Copyright (c) 2015 by Contributors * Copyright (c) 2017 Microsoft * Licensed under The MIT License [see LICENSE for details] * \file multi_proposal-inl.h * \brief MultiProposal Operator * \author Piotr Teterwak, Bing Xu, Jian Guo, Xizhou Zhu */ #ifndef MXNET_OPERATOR_CONTRIB_PROPOSAL_INL_H_ #define MXNET_OPERATOR_CONTRIB_PROPOSAL_INL_H_ #include #include #include #include #include #include #include #include #include #include #include "../operator_common.h" #include "../mshadow_op.h" // extend NumericalParam namespace mxnet { namespace op { /*! * \brief structure for numerical tuple input * \tparam VType data type of param */ template struct NumericalParam { NumericalParam() {} explicit NumericalParam(VType *begin, VType *end) { int32_t size = static_cast(end - begin); info.resize(size); for (int i = 0; i < size; ++i) { info[i] = *(begin + i); } } inline size_t ndim() const { return info.size(); } std::vector info; }; template inline std::istream &operator>>(std::istream &is, NumericalParam ¶m) { while (true) { char ch = is.get(); if (ch == '(') break; if (!isspace(ch)) { is.setstate(std::ios::failbit); return is; } } VType idx; std::vector tmp; // deal with empty case size_t pos = is.tellg(); char ch = is.get(); if (ch == ')') { param.info = tmp; return is; } is.seekg(pos); // finish deal while (is >> idx) { tmp.push_back(idx); char ch; do { ch = is.get(); } while (isspace(ch)); if (ch == ',') { while (true) { ch = is.peek(); if (isspace(ch)) { is.get(); continue; } if (ch == ')') { is.get(); break; } break; } if (ch == ')') break; } else if (ch == ')') { break; } else { is.setstate(std::ios::failbit); return is; } } param.info = tmp; return is; } template inline std::ostream &operator<<(std::ostream &os, const NumericalParam ¶m) { os << '('; for (index_t i = 0; i < param.info.size(); ++i) { if (i != 0) os << ','; os << param.info[i]; } // python style tuple if (param.info.size() == 1) os << ','; os << ')'; return os; } } // namespace op } // namespace mxnet namespace mxnet { namespace op { namespace proposal { enum MultiProposalOpInputs {kClsProb, kBBoxPred, kImInfo}; enum MultiProposalOpOutputs {kOut, kScore}; enum MultiProposalForwardResource {kTempResource}; } // proposal struct MultiProposalParam : public dmlc::Parameter { int rpn_pre_nms_top_n; int rpn_post_nms_top_n; float threshold; int rpn_min_size; NumericalParam scales; NumericalParam ratios; int feature_stride; bool output_score; bool iou_loss; DMLC_DECLARE_PARAMETER(MultiProposalParam) { float tmp[] = {0, 0, 0, 0}; DMLC_DECLARE_FIELD(rpn_pre_nms_top_n).set_default(6000) .describe("Number of top scoring boxes to keep after applying NMS to RPN proposals"); DMLC_DECLARE_FIELD(rpn_post_nms_top_n).set_default(300) .describe("Overlap threshold used for non-maximum" "suppresion(suppress boxes with IoU >= this threshold"); DMLC_DECLARE_FIELD(threshold).set_default(0.7) .describe("NMS value, below which to suppress."); DMLC_DECLARE_FIELD(rpn_min_size).set_default(16) .describe("Minimum height or width in proposal"); tmp[0] = 4.0f; tmp[1] = 8.0f; tmp[2] = 16.0f; tmp[3] = 32.0f; DMLC_DECLARE_FIELD(scales).set_default(NumericalParam(tmp, tmp + 4)) .describe("Used to generate anchor windows by enumerating scales"); tmp[0] = 0.5f; tmp[1] = 1.0f; tmp[2] = 2.0f; DMLC_DECLARE_FIELD(ratios).set_default(NumericalParam(tmp, tmp + 3)) .describe("Used to generate anchor windows by enumerating ratios"); DMLC_DECLARE_FIELD(feature_stride).set_default(16) .describe("The size of the receptive field each unit in the convolution layer of the rpn," "for example the product of all stride's prior to this layer."); DMLC_DECLARE_FIELD(output_score).set_default(false) .describe("Add score to outputs"); DMLC_DECLARE_FIELD(iou_loss).set_default(false) .describe("Usage of IoU Loss"); } }; template Operator *CreateOp(MultiProposalParam param); #if DMLC_USE_CXX11 class MultiProposalProp : public OperatorProperty { public: void Init(const std::vector >& kwargs) override { param_.Init(kwargs); } std::map GetParams() const override { return param_.__DICT__(); } bool InferShape(std::vector *in_shape, std::vector *out_shape, std::vector *aux_shape) const override { using namespace mshadow; CHECK_EQ(in_shape->size(), 3) << "Input:[cls_prob, bbox_pred, im_info]"; const TShape &dshape = in_shape->at(proposal::kClsProb); if (dshape.ndim() == 0) return false; Shape<4> bbox_pred_shape; bbox_pred_shape = Shape4(dshape[0], dshape[1] * 2, dshape[2], dshape[3]); SHAPE_ASSIGN_CHECK(*in_shape, proposal::kBBoxPred, bbox_pred_shape); Shape<2> im_info_shape; im_info_shape = Shape2(dshape[0], 3); SHAPE_ASSIGN_CHECK(*in_shape, proposal::kImInfo, im_info_shape); out_shape->clear(); // output out_shape->push_back(Shape2(dshape[0] * param_.rpn_post_nms_top_n, 5)); // score out_shape->push_back(Shape2(dshape[0] * param_.rpn_post_nms_top_n, 1)); return true; } OperatorProperty* Copy() const override { auto ptr = new MultiProposalProp(); ptr->param_ = param_; return ptr; } std::string TypeString() const override { return "_contrib_MultiProposal"; } std::vector ForwardResource( const std::vector &in_shape) const override { return {ResourceRequest::kTempSpace}; } std::vector DeclareBackwardDependency( const std::vector &out_grad, const std::vector &in_data, const std::vector &out_data) const override { return {}; } int NumVisibleOutputs() const override { if (param_.output_score) { return 2; } else { return 1; } } int NumOutputs() const override { return 2; } std::vector ListArguments() const override { return {"cls_prob", "bbox_pred", "im_info"}; } std::vector ListOutputs() const override { return {"output", "score"}; } Operator* CreateOperator(Context ctx) const override; private: MultiProposalParam param_; }; // class MultiProposalProp #endif // DMLC_USE_CXX11 } // namespace op } // namespace mxnet //======================== // Anchor Generation Utils //======================== namespace mxnet { namespace op { namespace utils { inline void _MakeAnchor(float w, float h, float x_ctr, float y_ctr, std::vector *out_anchors) { out_anchors->push_back(x_ctr - 0.5f * (w - 1.0f)); out_anchors->push_back(y_ctr - 0.5f * (h - 1.0f)); out_anchors->push_back(x_ctr + 0.5f * (w - 1.0f)); out_anchors->push_back(y_ctr + 0.5f * (h - 1.0f)); out_anchors->push_back(0.0f); } inline void _Transform(float scale, float ratio, const std::vector& base_anchor, std::vector *out_anchors) { float w = base_anchor[2] - base_anchor[1] + 1.0f; float h = base_anchor[3] - base_anchor[1] + 1.0f; float x_ctr = base_anchor[0] + 0.5 * (w - 1.0f); float y_ctr = base_anchor[1] + 0.5 * (h - 1.0f); float size = w * h; float size_ratios = std::floor(size / ratio); float new_w = std::floor(std::sqrt(size_ratios) + 0.5f) * scale; float new_h = std::floor((new_w / scale * ratio) + 0.5f) * scale; _MakeAnchor(new_w, new_h, x_ctr, y_ctr, out_anchors); } // out_anchors must have shape (n, 5), where n is ratios.size() * scales.size() inline void GenerateAnchors(const std::vector& base_anchor, const std::vector& ratios, const std::vector& scales, std::vector *out_anchors) { for (size_t j = 0; j < ratios.size(); ++j) { for (size_t k = 0; k < scales.size(); ++k) { _Transform(scales[k], ratios[j], base_anchor, out_anchors); } } } } // namespace utils } // namespace op } // namespace mxnet #endif // MXNET_OPERATOR_CONTRIB_PROPOSAL_INL_H_ ================================================ FILE: dff_rfcn/operator_cxx/multi_proposal.cc ================================================ /*! * Copyright (c) 2017 Microsoft * Licensed under The MIT License [see LICENSE for details] * \file multi_proposal.cc * \brief * \author Xizhou Zhu */ #include "./multi_proposal-inl.h" namespace mxnet { namespace op { template class MultiProposalOp : public Operator{ public: explicit MultiProposalOp(MultiProposalParam param) { this->param_ = param; } virtual void Forward(const OpContext &ctx, const std::vector &in_data, const std::vector &req, const std::vector &out_data, const std::vector &aux_states) { LOG(FATAL) << "not implemented"; } virtual void Backward(const OpContext &ctx, const std::vector &out_grad, const std::vector &in_data, const std::vector &out_data, const std::vector &req, const std::vector &in_grad, const std::vector &aux_states) { LOG(FATAL) << "not implemented"; } private: MultiProposalParam param_; }; // class MultiProposalOp template<> Operator *CreateOp(MultiProposalParam param) { return new MultiProposalOp(param); } Operator* MultiProposalProp::CreateOperator(Context ctx) const { DO_BIND_DISPATCH(CreateOp, param_); } DMLC_REGISTER_PARAMETER(MultiProposalParam); MXNET_REGISTER_OP_PROPERTY(_contrib_MultiProposal, MultiProposalProp) .describe("Generate region proposals via RPN") .add_argument("cls_score", "NDArray-or-Symbol", "Score of how likely proposal is object.") .add_argument("bbox_pred", "NDArray-or-Symbol", "BBox Predicted deltas from anchors for proposals") .add_argument("im_info", "NDArray-or-Symbol", "Image size and scale.") .add_arguments(MultiProposalParam::__FIELDS__()); } // namespace op } // namespace mxnet ================================================ FILE: dff_rfcn/operator_cxx/multi_proposal.cu ================================================ /*! * Copyright (c) 2015 by Contributors * Copyright (c) 2017 Microsoft * Licensed under The MIT License [see LICENSE for details] * \file multi_proposal.cu * \brief MultiProposal Operator * \author Shaoqing Ren, Xizhou Zhu, Jian Guo */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "../operator_common.h" #include "../mshadow_op.h" #include "./multi_proposal-inl.h" #define DIVUP(m, n) ((m) / (n) + ((m) % (n) > 0)) #define FRCNN_CUDA_CHECK(condition) \ /* Code block avoids redefinition of cudaError_t error */ \ do { \ cudaError_t error = condition; \ CHECK_EQ(error, cudaSuccess) << " " << cudaGetErrorString(error); \ } while (0) namespace mshadow { namespace cuda { namespace multi_proposal { // scores are (b, 2 * anchor, h, w) // workspace_proposals are (b, h * w * anchor, 5) // w defines "x" and h defines "y" // count should be total anchors numbers, h * w * anchors template __global__ void ProposalGridKernel(const int count, const int num_anchors, const int height, const int width, const int feature_stride, const Dtype* scores, Dtype* workspace_proposals) { for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < count; index += blockDim.x * gridDim.x) { int a = index % num_anchors; int w = (index / num_anchors) % width; int h = (index / num_anchors / width) % height; int b = index / num_anchors / width / height; workspace_proposals[index * 5 + 0] = workspace_proposals[a * 5 + 0] + w * feature_stride; workspace_proposals[index * 5 + 1] = workspace_proposals[a * 5 + 1] + h * feature_stride; workspace_proposals[index * 5 + 2] = workspace_proposals[a * 5 + 2] + w * feature_stride; workspace_proposals[index * 5 + 3] = workspace_proposals[a * 5 + 3] + h * feature_stride; workspace_proposals[index * 5 + 4] = scores[((b * (2 * num_anchors) + a + num_anchors) * height + h) * width + w]; //workspace_proposals[index * 5 + 4] = scores[(a * height + h) * width + w]; } } // boxes are (b, h * w * anchor, 5) // deltas are (b, 4 * anchor, h, w) // out_pred_boxes are (b, h * w * anchor, 5) // count should be total anchors numbers, b * h * w * anchors // in-place write: boxes and out_pred_boxes are the same location template __global__ void BBoxPredKernel(const int count, const int num_anchors, const int feat_height, const int feat_width, const int feature_stride, const Dtype* im_infos, const Dtype* boxes, const Dtype* deltas, Dtype* out_pred_boxes) { for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < count; index += blockDim.x * gridDim.x) { int a = index % num_anchors; int w = (index / num_anchors) % feat_width; int h = (index / num_anchors / feat_width) % feat_height; int b = index / num_anchors / feat_width / feat_height; float im_height = im_infos[b * 3]; float im_width = im_infos[b * 3 + 1]; int real_height = static_cast(im_height / feature_stride); int real_width = static_cast(im_width / feature_stride); float width = boxes[index * 5 + 2] - boxes[index * 5 + 0] + 1.0f; float height = boxes[index * 5 + 3] - boxes[index * 5 + 1] + 1.0f; float ctr_x = boxes[index * 5 + 0] + 0.5f * (width - 1.0f); float ctr_y = boxes[index * 5 + 1] + 0.5f * (height - 1.0f); int ba = (b * num_anchors + a); float dx = deltas[((ba * 4) * feat_height + h) * feat_width + w]; float dy = deltas[((ba * 4 + 1) * feat_height + h) * feat_width + w]; float dw = deltas[((ba * 4 + 2) * feat_height + h) * feat_width + w]; float dh = deltas[((ba * 4 + 3) * feat_height + h) * feat_width + w]; float pred_ctr_x = dx * width + ctr_x; float pred_ctr_y = dy * height + ctr_y; float pred_w = exp(dw) * width; float pred_h = exp(dh) * height; float pred_x1 = pred_ctr_x - 0.5f * (pred_w - 1.0f); float pred_y1 = pred_ctr_y - 0.5f * (pred_h - 1.0f); float pred_x2 = pred_ctr_x + 0.5f * (pred_w - 1.0f); float pred_y2 = pred_ctr_y + 0.5f * (pred_h - 1.0f); pred_x1 = max(min(pred_x1, im_width - 1.0f), 0.0f); pred_y1 = max(min(pred_y1, im_height - 1.0f), 0.0f); pred_x2 = max(min(pred_x2, im_width - 1.0f), 0.0f); pred_y2 = max(min(pred_y2, im_height - 1.0f), 0.0f); out_pred_boxes[index * 5 + 0] = pred_x1; out_pred_boxes[index * 5 + 1] = pred_y1; out_pred_boxes[index * 5 + 2] = pred_x2; out_pred_boxes[index * 5 + 3] = pred_y2; if (h >= real_height || w >= real_width) { out_pred_boxes[index * 5 + 4] = -1.0f; } } } // boxes are (b, h * w * anchor, 5) // deltas are (b, 4 * anchor, h, w) // out_pred_boxes are (b, h * w * anchor, 5) // count should be total anchors numbers, b * h * w * anchors // in-place write: boxes and out_pred_boxes are the same location template __global__ void IoUPredKernel(const int count, const int num_anchors, const int feat_height, const int feat_width, const int feature_stride, const Dtype* im_infos, const Dtype* boxes, const Dtype* deltas, Dtype* out_pred_boxes) { for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < count; index += blockDim.x * gridDim.x) { int a = index % num_anchors; int w = (index / num_anchors) % feat_width; int h = (index / num_anchors / feat_width) % feat_height; int b = index / num_anchors / feat_width / feat_height; float im_height = im_infos[b * 3]; float im_width = im_infos[b * 3 + 1]; int real_height = static_cast(im_height / feature_stride); int real_width = static_cast(im_width / feature_stride); float x1 = boxes[index * 5 + 0]; float y1 = boxes[index * 5 + 1]; float x2 = boxes[index * 5 + 2]; float y2 = boxes[index * 5 + 3]; int ba = (b * num_anchors + a); float dx1 = deltas[((ba * 4) * feat_height + h) * feat_width + w]; float dy1 = deltas[((ba * 4 + 1) * feat_height + h) * feat_width + w]; float dx2 = deltas[((ba * 4 + 2) * feat_height + h) * feat_width + w]; float dy2 = deltas[((ba * 4 + 3) * feat_height + h) * feat_width + w]; float pred_x1 = max(min(x1 + dx1, im_width - 1.0f), 0.0f); float pred_y1 = max(min(y1 + dy1, im_height - 1.0f), 0.0f); float pred_x2 = max(min(x2 + dx2, im_width - 1.0f), 0.0f); float pred_y2 = max(min(y2 + dy2, im_height - 1.0f), 0.0f); out_pred_boxes[index * 5 + 0] = pred_x1; out_pred_boxes[index * 5 + 1] = pred_y1; out_pred_boxes[index * 5 + 2] = pred_x2; out_pred_boxes[index * 5 + 3] = pred_y2; if (h >= real_height || w >= real_width) { out_pred_boxes[index * 5 + 4] = -1.0f; } } } // filter box with stride less than rpn_min_size // filter: set score to zero // dets (b, n, 5) template __global__ void FilterBoxKernel(const int count, const int count_anchors, const float original_min_size, const Dtype* im_infos, Dtype* dets) { for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < count; index += blockDim.x * gridDim.x) { int b = index / count_anchors; float iw = dets[index * 5 + 2] - dets[index * 5 + 0] + 1.0f; float ih = dets[index * 5 + 3] - dets[index * 5 + 1] + 1.0f; float min_size = original_min_size * im_infos[b * 3 + 2]; if (iw < min_size || ih < min_size) { dets[index * 5 + 0] -= min_size / 2; dets[index * 5 + 1] -= min_size / 2; dets[index * 5 + 2] += min_size / 2; dets[index * 5 + 3] += min_size / 2; dets[index * 5 + 4] = -1.0f; } } } // copy score and init order // dets (n, 5); score (n, ); order (n, ) // count should be n (total anchors or proposals) template __global__ void CopyScoreKernel(const int count, const Dtype* dets, Dtype* score, int* order) { for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < count; index += blockDim.x * gridDim.x) { score[index] = dets[index * 5 + 4]; order[index] = index; } } // reorder proposals according to order and keep the top_n proposals // prev_dets (n, 5); order (n, ); dets (n, 5) // count should be output anchor numbers (top_n) template __global__ void ReorderProposalsKernel(const int count, const Dtype* prev_dets, const int* order, Dtype* dets) { for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < count; index += blockDim.x * gridDim.x) { const int order_i = order[index]; for (int j = 0; j < 5; j ++) { dets[index * 5 + j] = prev_dets[order_i * 5 + j]; } } } __device__ inline float devIoU(float const * const a, float const * const b) { float left = max(a[0], b[0]), right = min(a[2], b[2]); float top = max(a[1], b[1]), bottom = min(a[3], b[3]); float width = max(right - left + 1, 0.f), height = max(bottom - top + 1, 0.f); float interS = width * height; float Sa = (a[2] - a[0] + 1) * (a[3] - a[1] + 1); float Sb = (b[2] - b[0] + 1) * (b[3] - b[1] + 1); return interS / (Sa + Sb - interS); } __global__ void nms_kernel(const int n_boxes, const float nms_overlap_thresh, const float *dev_boxes, uint64_t *dev_mask) { const int threadsPerBlock = sizeof(uint64_t) * 8; const int row_start = blockIdx.y; const int col_start = blockIdx.x; // if (row_start > col_start) return; const int row_size = min(n_boxes - row_start * threadsPerBlock, threadsPerBlock); const int col_size = min(n_boxes - col_start * threadsPerBlock, threadsPerBlock); __shared__ float block_boxes[threadsPerBlock * 5]; if (threadIdx.x < col_size) { block_boxes[threadIdx.x * 5 + 0] = dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 0]; block_boxes[threadIdx.x * 5 + 1] = dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 1]; block_boxes[threadIdx.x * 5 + 2] = dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 2]; block_boxes[threadIdx.x * 5 + 3] = dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 3]; block_boxes[threadIdx.x * 5 + 4] = dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 4]; } __syncthreads(); if (threadIdx.x < row_size) { const int cur_box_idx = threadsPerBlock * row_start + threadIdx.x; const float *cur_box = dev_boxes + cur_box_idx * 5; int i = 0; uint64_t t = 0; int start = 0; if (row_start == col_start) { start = threadIdx.x + 1; } for (i = start; i < col_size; i++) { if (devIoU(cur_box, block_boxes + i * 5) > nms_overlap_thresh) { t |= 1ULL << i; } } const int col_blocks = DIVUP(n_boxes, threadsPerBlock); dev_mask[cur_box_idx * col_blocks + col_start] = t; } } void _nms(const mshadow::Tensor& boxes, const float nms_overlap_thresh, int *keep, int *num_out) { const int threadsPerBlock = sizeof(uint64_t) * 8; const int boxes_num = boxes.size(0); const int boxes_dim = boxes.size(1); float* boxes_dev = boxes.dptr_; uint64_t* mask_dev = NULL; const int col_blocks = DIVUP(boxes_num, threadsPerBlock); FRCNN_CUDA_CHECK(cudaMalloc(&mask_dev, boxes_num * col_blocks * sizeof(uint64_t))); dim3 blocks(DIVUP(boxes_num, threadsPerBlock), DIVUP(boxes_num, threadsPerBlock)); dim3 threads(threadsPerBlock); nms_kernel<<>>(boxes_num, nms_overlap_thresh, boxes_dev, mask_dev); FRCNN_CUDA_CHECK(cudaPeekAtLastError()); std::vector mask_host(boxes_num * col_blocks); FRCNN_CUDA_CHECK(cudaMemcpy(&mask_host[0], mask_dev, sizeof(uint64_t) * boxes_num * col_blocks, cudaMemcpyDeviceToHost)); std::vector remv(col_blocks); memset(&remv[0], 0, sizeof(uint64_t) * col_blocks); int num_to_keep = 0; for (int i = 0; i < boxes_num; i++) { int nblock = i / threadsPerBlock; int inblock = i % threadsPerBlock; if (!(remv[nblock] & (1ULL << inblock))) { keep[num_to_keep++] = i; uint64_t *p = &mask_host[0] + i * col_blocks; for (int j = nblock; j < col_blocks; j++) { remv[j] |= p[j]; } } } *num_out = num_to_keep; FRCNN_CUDA_CHECK(cudaFree(mask_dev)); } // copy proposals to output // dets (top_n, 5); keep (top_n, ); out (top_n, ) // count should be top_n (total anchors or proposals) template __global__ void PrepareOutput(const int count, const Dtype* dets, const int* keep, const int out_size, const int image_index, Dtype* out, Dtype* score) { for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < count; index += blockDim.x * gridDim.x) { out[index * 5] = image_index; if (index < out_size) { int keep_i = keep[index]; for (int j = 0; j < 4; ++j) { out[index * 5 + j + 1] = dets[keep_i * 5 + j]; } score[index] = dets[keep_i * 5 + 4]; } else { int keep_i = keep[index % out_size]; for (int j = 0; j < 4; ++j) { out[index * 5 + j + 1] = dets[keep_i * 5 + j]; } score[index] = dets[keep_i * 5 + 4]; } } } } // namespace multi_proposal } // namespace cuda } // namespace mshadow namespace mxnet { namespace op { template class MultiProposalGPUOp : public Operator{ public: explicit MultiProposalGPUOp(MultiProposalParam param) { this->param_ = param; } virtual void Forward(const OpContext &ctx, const std::vector &in_data, const std::vector &req, const std::vector &out_data, const std::vector &aux_states) { using namespace mshadow; using namespace mshadow::expr; using namespace mshadow::cuda; using namespace mshadow::cuda::multi_proposal; CHECK_EQ(in_data.size(), 3); CHECK_EQ(out_data.size(), 2); CHECK_GT(req.size(), 1); CHECK_EQ(req[proposal::kOut], kWriteTo); /*CHECK_EQ(in_data[proposal::kClsProb].shape_[0], 1) << "Sorry, multiple images each device is not implemented.";*/ Stream *s = ctx.get_stream(); Tensor scores = in_data[proposal::kClsProb].get(s); Tensor bbox_deltas = in_data[proposal::kBBoxPred].get(s); Tensor im_info = in_data[proposal::kImInfo].get(s); Tensor out = out_data[proposal::kOut].get(s); Tensor out_score = out_data[proposal::kScore].get(s); int num_images = scores.size(0); int num_anchors = scores.size(1) / 2; int height = scores.size(2); int width = scores.size(3); int count_anchors = num_anchors * height * width; // count of total anchors int count = num_images * count_anchors; // set to -1 for max int rpn_pre_nms_top_n = (param_.rpn_pre_nms_top_n > 0) ? param_.rpn_pre_nms_top_n : count_anchors; rpn_pre_nms_top_n = std::min(rpn_pre_nms_top_n, count_anchors); int rpn_post_nms_top_n = std::min(param_.rpn_post_nms_top_n, rpn_pre_nms_top_n); // Generate first anchors based on base anchor std::vector base_anchor(4); base_anchor[0] = 0.0; base_anchor[1] = 0.0; base_anchor[2] = param_.feature_stride - 1.0; base_anchor[3] = param_.feature_stride - 1.0; CHECK_EQ(num_anchors, param_.ratios.info.size() * param_.scales.info.size()); std::vector anchors; utils::GenerateAnchors(base_anchor, param_.ratios.info, param_.scales.info, &anchors); // Copy generated anchors to GPU float* workspace_proposals_ptr = NULL; FRCNN_CUDA_CHECK(cudaMalloc(&workspace_proposals_ptr, sizeof(float) * num_images * count_anchors * 5)); Tensor workspace_proposals(workspace_proposals_ptr, Shape3(num_images, count_anchors, 5)); FRCNN_CUDA_CHECK(cudaMemcpy(workspace_proposals.dptr_, &anchors[0], sizeof(float) * anchors.size(), cudaMemcpyHostToDevice)); // Copy proposals to a mesh grid dim3 dimGrid((count + kMaxThreadsPerBlock - 1) / kMaxThreadsPerBlock); dim3 dimBlock(kMaxThreadsPerBlock); CheckLaunchParam(dimGrid, dimBlock, "ProposalGrid"); ProposalGridKernel<<>>( count, num_anchors, height, width, param_.feature_stride, scores.dptr_, workspace_proposals.dptr_); FRCNN_CUDA_CHECK(cudaPeekAtLastError()); // Transform anchors and bbox_deltas into bboxes CheckLaunchParam(dimGrid, dimBlock, "BBoxPred"); if (param_.iou_loss) { IoUPredKernel<<>>( count, num_anchors, height, width, param_.feature_stride, im_info.dptr_, workspace_proposals.dptr_, bbox_deltas.dptr_, workspace_proposals.dptr_); } else { BBoxPredKernel<<>>( count, num_anchors, height, width, param_.feature_stride, im_info.dptr_, workspace_proposals.dptr_, bbox_deltas.dptr_, workspace_proposals.dptr_); } FRCNN_CUDA_CHECK(cudaPeekAtLastError()); // filter boxes with less than rpn_min_size CheckLaunchParam(dimGrid, dimBlock, "FilterBox"); FilterBoxKernel<<>>( count, count_anchors, param_.rpn_min_size, im_info.dptr_, workspace_proposals.dptr_); FRCNN_CUDA_CHECK(cudaPeekAtLastError()); dimGrid = dim3((count_anchors + kMaxThreadsPerBlock - 1) / kMaxThreadsPerBlock); dimBlock = dim3(kMaxThreadsPerBlock); // Copy score to a continuous memory float* score_ptr = NULL; FRCNN_CUDA_CHECK(cudaMalloc(&score_ptr, sizeof(float) * count_anchors)); Tensor score(score_ptr, Shape1(count_anchors)); int* order_ptr = NULL; FRCNN_CUDA_CHECK(cudaMalloc(&order_ptr, sizeof(int) * count_anchors)); Tensor order(order_ptr, Shape1(count_anchors)); float* workspace_ordered_proposals_ptr = NULL; FRCNN_CUDA_CHECK(cudaMalloc(&workspace_ordered_proposals_ptr, sizeof(float) * rpn_pre_nms_top_n * 5)); Tensor workspace_ordered_proposals(workspace_ordered_proposals_ptr, Shape2(rpn_pre_nms_top_n, 5)); int* keep; FRCNN_CUDA_CHECK(cudaMalloc(&keep, sizeof(int) * rpn_pre_nms_top_n)); for (int b = 0; b < num_images; b++) { CheckLaunchParam(dimGrid, dimBlock, "CopyScore"); CopyScoreKernel << > >( count_anchors, workspace_proposals.dptr_ + b * count_anchors * 5, score.dptr_, order.dptr_); FRCNN_CUDA_CHECK(cudaPeekAtLastError()); // argsort score, save order thrust::stable_sort_by_key(thrust::device, score.dptr_, score.dptr_ + score.size(0), order.dptr_, thrust::greater()); FRCNN_CUDA_CHECK(cudaPeekAtLastError()); // Reorder proposals according to order dimGrid.x = (rpn_pre_nms_top_n + kMaxThreadsPerBlock - 1) / kMaxThreadsPerBlock; CheckLaunchParam(dimGrid, dimBlock, "ReorderProposals"); ReorderProposalsKernel << > >( rpn_pre_nms_top_n, workspace_proposals.dptr_ + b * count_anchors * 5, order.dptr_, workspace_ordered_proposals.dptr_); FRCNN_CUDA_CHECK(cudaPeekAtLastError()); // perform nms std::vector _keep(workspace_ordered_proposals.size(0)); int out_size = 0; _nms(workspace_ordered_proposals, param_.threshold, &_keep[0], &out_size); // copy nms result to gpu FRCNN_CUDA_CHECK(cudaMemcpy(keep, &_keep[0], sizeof(int) * _keep.size(), cudaMemcpyHostToDevice)); // copy results after nms dimGrid.x = (rpn_post_nms_top_n + kMaxThreadsPerBlock - 1) / kMaxThreadsPerBlock; CheckLaunchParam(dimGrid, dimBlock, "PrepareOutput"); PrepareOutput << > >( rpn_post_nms_top_n, workspace_ordered_proposals.dptr_, keep, out_size, b, out.dptr_ + b * rpn_post_nms_top_n * 5, out_score.dptr_ + b * rpn_post_nms_top_n); FRCNN_CUDA_CHECK(cudaPeekAtLastError()); } // free temporary memory FRCNN_CUDA_CHECK(cudaFree(keep)); FRCNN_CUDA_CHECK(cudaFree(workspace_ordered_proposals_ptr)); FRCNN_CUDA_CHECK(cudaFree(workspace_proposals_ptr)); FRCNN_CUDA_CHECK(cudaFree(score_ptr)); FRCNN_CUDA_CHECK(cudaFree(order_ptr)); } virtual void Backward(const OpContext &ctx, const std::vector &out_grad, const std::vector &in_data, const std::vector &out_data, const std::vector &req, const std::vector &in_grad, const std::vector &aux_states) { using namespace mshadow; using namespace mshadow::expr; CHECK_EQ(in_grad.size(), 3); Stream *s = ctx.get_stream(); Tensor gscores = in_grad[proposal::kClsProb].get(s); Tensor gbbox = in_grad[proposal::kBBoxPred].get(s); Tensor ginfo = in_grad[proposal::kImInfo].get(s); // can not assume the grad would be zero Assign(gscores, req[proposal::kClsProb], 0); Assign(gbbox, req[proposal::kBBoxPred], 0); Assign(ginfo, req[proposal::kImInfo], 0); } private: MultiProposalParam param_; }; // class MultiProposalGPUOp template<> Operator* CreateOp(MultiProposalParam param) { return new MultiProposalGPUOp(param); } } // namespace op } // namespace mxnet ================================================ FILE: dff_rfcn/operator_cxx/psroi_pooling-inl.h ================================================ /*! * Copyright (c) 2017 by Contributors * Copyright (c) 2017 Microsoft * Licensed under The MIT License [see LICENSE for details] * \file psroi_pooling-inl.h * \brief psroi pooling operator and symbol * \author Yi Li, Tairui Chen, Guodong Zhang, Jifeng Dai */ #ifndef MXNET_OPERATOR_PSROI_POOLING_INL_H_ #define MXNET_OPERATOR_PSROI_POOLING_INL_H_ #include #include #include #include #include #include #include #include "../mshadow_op.h" #include "../operator_common.h" namespace mxnet { namespace op { // Declare enumeration of input order to make code more intuitive. // These enums are only visible within this header namespace psroipool { enum PSROIPoolingOpInputs {kData, kBox}; enum PSROIPoolingOpOutputs {kOut, kMappingChannel}; } // psroipool struct PSROIPoolingParam : public dmlc::Parameter { // TShape pooled_size; float spatial_scale; int output_dim; int pooled_size; int group_size; DMLC_DECLARE_PARAMETER(PSROIPoolingParam) { DMLC_DECLARE_FIELD(spatial_scale).set_range(0.0, 1.0) .describe("Ratio of input feature map height (or w) to raw image height (or w). " "Equals the reciprocal of total stride in convolutional layers"); DMLC_DECLARE_FIELD(output_dim).describe("fix output dim"); DMLC_DECLARE_FIELD(pooled_size).describe("fix pooled size"); DMLC_DECLARE_FIELD(group_size).set_default(0).describe("fix group size"); } }; template class PSROIPoolingOp : public Operator { public: explicit PSROIPoolingOp(PSROIPoolingParam p) { this->param_ = p; } virtual void Forward(const OpContext &ctx, const std::vector &in_data, const std::vector &req, const std::vector &out_data, const std::vector &aux_args) { using namespace mshadow; size_t expected = 2; CHECK_EQ(in_data.size(), expected); CHECK_EQ(out_data.size(), expected); CHECK_EQ(out_data[psroipool::kOut].shape_[0], in_data[psroipool::kBox].shape_[0]); CHECK_EQ(out_data[psroipool::kMappingChannel].shape_[0], in_data[psroipool::kBox].shape_[0]); Stream *s = ctx.get_stream(); Tensor data = in_data[psroipool::kData].get(s); Tensor bbox = in_data[psroipool::kBox].get(s); Tensor out = out_data[psroipool::kOut].get(s); Tensor mapping_channel = out_data[psroipool::kMappingChannel].get(s); CHECK_EQ(data.CheckContiguous(), true); CHECK_EQ(bbox.CheckContiguous(), true); CHECK_EQ(out.CheckContiguous(), true); CHECK_EQ(mapping_channel.CheckContiguous(), true); out = -FLT_MAX; mapping_channel = -1.0f; PSROIPoolForward(out, data, bbox, mapping_channel, param_.spatial_scale, param_.output_dim, param_.group_size); } virtual void Backward(const OpContext &ctx, const std::vector &out_grad, const std::vector &in_data, const std::vector &out_data, const std::vector &req, const std::vector &in_grad, const std::vector &aux_args) { using namespace mshadow; size_t expected = 2; CHECK_EQ(in_data.size(), expected); CHECK_EQ(out_data.size(), expected); CHECK_EQ(out_grad[psroipool::kOut].shape_[0], in_data[psroipool::kBox].shape_[0]); CHECK_EQ(out_data[psroipool::kMappingChannel].shape_[0], in_data[psroipool::kBox].shape_[0]); CHECK_NE(req[psroipool::kData], kWriteInplace) << "ROIPooling: Backward doesn't support kWriteInplace."; CHECK_NE(req[psroipool::kBox], kWriteInplace) << "ROIPooling: Backward doesn't support kWriteInplace."; Stream *s = ctx.get_stream(); Tensor grad_out = out_grad[psroipool::kOut].get(s); Tensor bbox = in_data[psroipool::kBox].get(s); Tensor mapping_channel = out_data[psroipool::kMappingChannel].get(s); Tensor grad_in = in_grad[psroipool::kData].get(s); Tensor grad_roi = in_grad[psroipool::kBox].get(s); CHECK_EQ(grad_out.CheckContiguous(), true); CHECK_EQ(bbox.CheckContiguous(), true); CHECK_EQ(mapping_channel.CheckContiguous(), true); CHECK_EQ(grad_in.CheckContiguous(), true); if (kAddTo == req[psroipool::kData] || kWriteTo == req[psroipool::kData]) { if (kWriteTo == req[psroipool::kData]) { grad_in = 0.0f; } PSROIPoolBackwardAcc(grad_in, grad_out, bbox, mapping_channel, param_.spatial_scale, param_.output_dim); } if (kWriteTo == req[psroipool::kBox]) { grad_roi = 0.0f; } } private: PSROIPoolingParam param_; }; // class PSROIPoolingOp // Decalre Factory function, used for dispatch specialization template Operator* CreateOp(PSROIPoolingParam param, int dtype); #if DMLC_USE_CXX11 class PSROIPoolingProp : public OperatorProperty { public: std::vector ListArguments() const override { return {"data", "rois"}; } std::vector ListOutputs() const override { return {"output", "maxidx"}; } int NumOutputs() const override { return 2; } int NumVisibleOutputs() const override { return 1; } void Init(const std::vector >& kwargs) override { param_.Init(kwargs); if (param_.group_size == 0) { param_.group_size = param_.pooled_size; } } std::map GetParams() const override { return param_.__DICT__(); } bool InferShape(std::vector *in_shape, std::vector *out_shape, std::vector *aux_shape) const override { using namespace mshadow; CHECK_EQ(in_shape->size(), 2) << "Input:[data, rois]"; // data: [batch_size, c, h, w] TShape dshape = in_shape->at(psroipool::kData); CHECK_EQ(dshape.ndim(), 4) << "data should be a 4D tensor"; // bbox: [num_rois, 5] TShape bshape = in_shape->at(psroipool::kBox); CHECK_EQ(bshape.ndim(), 2) << "bbox should be a 2D tensor of shape [batch, 5]"; CHECK_EQ(bshape[1], 5) << "bbox should be a 2D tensor of shape [batch, 5]"; // out: [num_rois, c, pooled_h, pooled_w] // mapping_channel: [num_rois, c, pooled_h, pooled_w] out_shape->clear(); out_shape->push_back( Shape4(bshape[0], param_.output_dim, param_.pooled_size, param_.pooled_size)); out_shape->push_back( Shape4(bshape[0], param_.output_dim, param_.pooled_size, param_.pooled_size)); return true; } bool InferType(std::vector *in_type, std::vector *out_type, std::vector *aux_type) const override { CHECK_EQ(in_type->size(), 2); int dtype = (*in_type)[0]; CHECK_EQ(dtype, (*in_type)[1]); CHECK_NE(dtype, -1) << "Input must have specified type"; out_type->clear(); out_type->push_back(dtype); out_type->push_back(dtype); return true; } OperatorProperty* Copy() const override { PSROIPoolingProp* psroi_pooling_sym = new PSROIPoolingProp(); psroi_pooling_sym->param_ = this->param_; return psroi_pooling_sym; } std::string TypeString() const override { return "_contrib_PSROIPooling"; } // decalre dependency and inplace optimization options std::vector DeclareBackwardDependency( const std::vector &out_grad, const std::vector &in_data, const std::vector &out_data) const override { return {out_grad[psroipool::kOut], in_data[psroipool::kBox], out_data[psroipool::kMappingChannel]}; } Operator* CreateOperator(Context ctx) const override { LOG(FATAL) << "Not Implemented."; return NULL; } Operator* CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const override; private: PSROIPoolingParam param_; }; // class PSROIPoolingProp #endif } // namespace op } // namespace mxnet #endif // MXNET_OPERATOR_PSROI_POOLING_INL_H_ ================================================ FILE: dff_rfcn/operator_cxx/psroi_pooling.cc ================================================ /*! * Copyright (c) 2017 by Contributors * Copyright (c) 2017 Microsoft * Licensed under The MIT License [see LICENSE for details] * \file psroi_pooling.cc * \brief psroi pooling operator * \author Yi Li, Tairui Chen, Guodong Zhang, Jifeng Dai */ #include "./psroi_pooling-inl.h" #include #include #include #include #include using std::max; using std::min; using std::floor; using std::ceil; namespace mshadow { template inline void PSROIPoolForward(const Tensor &out, const Tensor &data, const Tensor &bbox, const Tensor &mapping_channel, const float spatial_scale_, const int output_dim_, const int group_size_) { // NOT_IMPLEMENTED; return; } template inline void PSROIPoolBackwardAcc(const Tensor &in_grad, const Tensor &out_grad, const Tensor &bbox, const Tensor &mapping_channel, const float spatial_scale_, const int output_dim_) { // NOT_IMPLEMENTED; return; } } // namespace mshadow namespace mxnet { namespace op { template<> Operator *CreateOp(PSROIPoolingParam param, int dtype) { Operator* op = NULL; MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { op = new PSROIPoolingOp(param); }); return op; } Operator *PSROIPoolingProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { std::vector out_shape, aux_shape; std::vector out_type, aux_type; CHECK(InferType(in_type, &out_type, &aux_type)); CHECK(InferShape(in_shape, &out_shape, &aux_shape)); DO_BIND_DISPATCH(CreateOp, param_, in_type->at(0)); } DMLC_REGISTER_PARAMETER(PSROIPoolingParam); MXNET_REGISTER_OP_PROPERTY(_contrib_PSROIPooling, PSROIPoolingProp) .describe("Performs region-of-interest pooling on inputs. Resize bounding box coordinates by " "spatial_scale and crop input feature maps accordingly. The cropped feature maps are pooled " "by max pooling to a fixed size output indicated by pooled_size. batch_size will change to " "the number of region bounding boxes after PSROIPooling") .add_argument("data", "Symbol", "Input data to the pooling operator, a 4D Feature maps") .add_argument("rois", "Symbol", "Bounding box coordinates, a 2D array of " "[[batch_index, x1, y1, x2, y2]]. (x1, y1) and (x2, y2) are top left and down right corners " "of designated region of interest. batch_index indicates the index of corresponding image " "in the input data") .add_arguments(PSROIPoolingParam::__FIELDS__()); } // namespace op } // namespace mxnet ================================================ FILE: dff_rfcn/operator_cxx/psroi_pooling.cu ================================================ /*! * Copyright (c) 2017 by Contributors * Copyright (c) 2017 Microsoft * Licensed under The MIT License [see LICENSE for details] * \file psroi_pooling.cu * \brief psroi pooling operator * \author Yi Li, Tairui Chen, Guodong Zhang, Jifeng Dai */ #include "./psroi_pooling-inl.h" #include #include #include #include #include "../../common/cuda_utils.h" #include "../mxnet_op.h" #define PSROIPOOLING_CUDA_CHECK(condition) \ /* Code block avoids redefinition of cudaError_t error */ \ do { \ cudaError_t error = condition; \ CHECK_EQ(error, cudaSuccess) << " " << cudaGetErrorString(error); \ } while (0) #define CUDA_KERNEL_LOOP(i, n) \ for (int i = blockIdx.x * blockDim.x + threadIdx.x; \ i < (n); \ i += blockDim.x * gridDim.x) namespace mshadow { namespace cuda { template __global__ void PSROIPoolForwardKernel( const int count, const DType* bottom_data, const DType spatial_scale, const int channels, const int height, const int width, const int pooled_height, const int pooled_width, const DType* bottom_rois, const int output_dim, const int group_size, DType* top_data, DType* mapping_channel) { CUDA_KERNEL_LOOP(index, count) { // The output is in order (n, ctop, ph, pw) int pw = index % pooled_width; int ph = (index / pooled_width) % pooled_height; int ctop = (index / pooled_width / pooled_height) % output_dim; int n = index / pooled_width / pooled_height / output_dim; // [start, end) interval for spatial sampling bottom_rois += n * 5; int roi_batch_ind = bottom_rois[0]; DType roi_start_w = static_cast(round(bottom_rois[1])) * spatial_scale; DType roi_start_h = static_cast(round(bottom_rois[2])) * spatial_scale; DType roi_end_w = static_cast(round(bottom_rois[3]) + 1.) * spatial_scale; DType roi_end_h = static_cast(round(bottom_rois[4]) + 1.) * spatial_scale; // Force too small ROIs to be 1x1 DType roi_width = max(roi_end_w - roi_start_w, 0.1); //avoid 0 DType roi_height = max(roi_end_h - roi_start_h, 0.1); // Compute w and h at bottom DType bin_size_h = roi_height / static_cast(pooled_height); DType bin_size_w = roi_width / static_cast(pooled_width); int hstart = floor(static_cast(ph) * bin_size_h + roi_start_h); int wstart = floor(static_cast(pw)* bin_size_w + roi_start_w); int hend = ceil(static_cast(ph + 1) * bin_size_h + roi_start_h); int wend = ceil(static_cast(pw + 1) * bin_size_w + roi_start_w); // Add roi offsets and clip to input boundaries hstart = min(max(hstart, 0), height); hend = min(max(hend, 0), height); wstart = min(max(wstart, 0),width); wend = min(max(wend, 0), width); bool is_empty = (hend <= hstart) || (wend <= wstart); int gw = floor(static_cast(pw)* group_size / pooled_width); int gh = floor(static_cast(ph)* group_size / pooled_height); gw = min(max(gw, 0), group_size - 1); gh = min(max(gh, 0), group_size - 1); int c = (ctop*group_size + gh)*group_size + gw; bottom_data += (roi_batch_ind * channels + c) * height * width; DType out_sum = 0; for (int h = hstart; h < hend; ++h){ for (int w = wstart; w < wend; ++w){ int bottom_index = h*width + w; out_sum += bottom_data[bottom_index]; } } DType bin_area = (hend - hstart)*(wend - wstart); top_data[index] = is_empty? (DType)0. : out_sum/bin_area; mapping_channel[index] = c; } } template inline void PSROIPoolForward(const Tensor &out, const Tensor &data, const Tensor &bbox, const Tensor &mapping_channel, const float spatial_scale, const int output_dim_, const int group_size_) { // LOG(INFO) << "PSROIPoolForward"; const DType *bottom_data = data.dptr_; const DType *bottom_rois = bbox.dptr_; DType *top_data = out.dptr_; DType *mapping_channel_ptr = mapping_channel.dptr_; const int count = out.shape_.Size(); const int channels = data.size(1); const int height = data.size(2); const int width = data.size(3); const int pooled_height = out.size(2); const int pooled_width = out.size(3); cudaStream_t stream = Stream::GetStream(out.stream_); PSROIPoolForwardKernel << > >( count, bottom_data, spatial_scale, channels, height, width, pooled_height, pooled_width, bottom_rois, output_dim_, group_size_, top_data, mapping_channel_ptr); PSROIPOOLING_CUDA_CHECK(cudaPeekAtLastError()); } template __global__ void PSROIPoolBackwardAccKernel( const int count, const DType* top_diff, const DType* mapping_channel, const int num_rois, const DType spatial_scale, const int channels, const int height, const int width, const int pooled_height, const int pooled_width, const int output_dim, DType* bottom_diff, const DType* bottom_rois) { CUDA_KERNEL_LOOP(index, count) { // The output is in order (n, ctop, ph, pw) int pw = index % pooled_width; int ph = (index / pooled_width) % pooled_height; int n = index / pooled_width / pooled_height / output_dim; // [start, end) interval for spatial sampling bottom_rois += n * 5; int roi_batch_ind = bottom_rois[0]; DType roi_start_w = static_cast(round(bottom_rois[1])) * spatial_scale; DType roi_start_h = static_cast(round(bottom_rois[2])) * spatial_scale; DType roi_end_w = static_cast(round(bottom_rois[3]) + 1.) * spatial_scale; DType roi_end_h = static_cast(round(bottom_rois[4]) + 1.) * spatial_scale; // Force too small ROIs to be 1x1 DType roi_width = max(roi_end_w - roi_start_w, 0.1); //avoid 0 DType roi_height = max(roi_end_h - roi_start_h, 0.1); // Compute w and h at bottom DType bin_size_h = roi_height / static_cast(pooled_height); DType bin_size_w = roi_width / static_cast(pooled_width); int hstart = floor(static_cast(ph)* bin_size_h + roi_start_h); int wstart = floor(static_cast(pw)* bin_size_w + roi_start_w); int hend = ceil(static_cast(ph + 1) * bin_size_h + roi_start_h); int wend = ceil(static_cast(pw + 1) * bin_size_w + roi_start_w); // Add roi offsets and clip to input boundaries hstart = min(max(hstart, 0), height); hend = min(max(hend, 0), height); wstart = min(max(wstart, 0), width); wend = min(max(wend, 0), width); bool is_empty = (hend <= hstart) || (wend <= wstart); // Compute c at bottom int c = mapping_channel[index]; DType* offset_bottom_diff = bottom_diff + (roi_batch_ind * channels + c) * height * width; DType bin_area = (hend - hstart)*(wend - wstart); DType diff_val = is_empty ? (DType)0. : top_diff[index] / bin_area; for (int h = hstart; h < hend; ++h){ for (int w = wstart; w < wend; ++w){ int bottom_index = h*width + w; // mxnet_gpu_atomic_add(diff_val, offset_bottom_diff + bottom_index); atomicAdd(offset_bottom_diff + bottom_index, diff_val); } } } } template inline void PSROIPoolBackwardAcc(const Tensor &in_grad, const Tensor &out_grad, const Tensor &bbox, const Tensor &mapping_channel, const float spatial_scale, const int output_dim_) { // LOG(INFO) << "PSROIPoolBackward"; const DType *top_diff = out_grad.dptr_; const DType *bottom_rois = bbox.dptr_; DType *bottom_diff = in_grad.dptr_; DType *mapping_channel_ptr = mapping_channel.dptr_; const int count = out_grad.shape_.Size(); const int num_rois = bbox.size(0); const int channels = in_grad.size(1); const int height = in_grad.size(2); const int width = in_grad.size(3); const int pooled_height = out_grad.size(2); const int pooled_width = out_grad.size(3); cudaStream_t stream = Stream::GetStream(in_grad.stream_); PSROIPoolBackwardAccKernel << > >( count, top_diff, mapping_channel_ptr, num_rois, spatial_scale, channels, height, width, pooled_height, pooled_width, output_dim_, bottom_diff, bottom_rois); PSROIPOOLING_CUDA_CHECK(cudaPeekAtLastError()); } } // namespace cuda template inline void PSROIPoolForward(const Tensor &out, const Tensor &data, const Tensor &bbox, const Tensor &mapping_channel, const float spatial_scale, const int output_dim_, const int group_size_) { cuda::PSROIPoolForward(out, data, bbox, mapping_channel, spatial_scale, output_dim_, group_size_); } template inline void PSROIPoolBackwardAcc(const Tensor &in_grad, const Tensor &out_grad, const Tensor &bbox, const Tensor &mapping_channel, const float spatial_scale, const int output_dim_) { cuda::PSROIPoolBackwardAcc(in_grad, out_grad, bbox, mapping_channel, spatial_scale, output_dim_); } } // namespace mshadow namespace mxnet { namespace op { template<> Operator* CreateOp(PSROIPoolingParam param, int dtype) { Operator* op = NULL; MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { op = new PSROIPoolingOp(param); }); return op; } } // namespace op } // namespace mxnet ================================================ FILE: dff_rfcn/operator_py/__init__.py ================================================ ================================================ FILE: dff_rfcn/operator_py/box_annotator_ohem.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong # -------------------------------------------------------- """ Proposal Target Operator selects foreground and background roi and assigns label, bbox_transform to them. """ import mxnet as mx import numpy as np from distutils.util import strtobool class BoxAnnotatorOHEMOperator(mx.operator.CustomOp): def __init__(self, num_classes, num_reg_classes, roi_per_img): super(BoxAnnotatorOHEMOperator, self).__init__() self._num_classes = num_classes self._num_reg_classes = num_reg_classes self._roi_per_img = roi_per_img def forward(self, is_train, req, in_data, out_data, aux): cls_score = in_data[0] bbox_pred = in_data[1] labels = in_data[2].asnumpy() bbox_targets = in_data[3] bbox_weights = in_data[4] per_roi_loss_cls = mx.nd.SoftmaxActivation(cls_score) + 1e-14 per_roi_loss_cls = per_roi_loss_cls.asnumpy() per_roi_loss_cls = per_roi_loss_cls[np.arange(per_roi_loss_cls.shape[0], dtype='int'), labels.astype('int')] per_roi_loss_cls = -1 * np.log(per_roi_loss_cls) per_roi_loss_cls = np.reshape(per_roi_loss_cls, newshape=(-1,)) per_roi_loss_bbox = bbox_weights * mx.nd.smooth_l1((bbox_pred - bbox_targets), scalar=1.0) per_roi_loss_bbox = mx.nd.sum(per_roi_loss_bbox, axis=1).asnumpy() top_k_per_roi_loss = np.argsort(per_roi_loss_cls + per_roi_loss_bbox) labels_ohem = labels labels_ohem[top_k_per_roi_loss[::-1][self._roi_per_img:]] = -1 bbox_weights_ohem = bbox_weights.asnumpy() bbox_weights_ohem[top_k_per_roi_loss[::-1][self._roi_per_img:]] = 0 labels_ohem = mx.nd.array(labels_ohem) bbox_weights_ohem = mx.nd.array(bbox_weights_ohem) for ind, val in enumerate([labels_ohem, bbox_weights_ohem]): self.assign(out_data[ind], req[ind], val) def backward(self, req, out_grad, in_data, out_data, in_grad, aux): for i in range(len(in_grad)): self.assign(in_grad[i], req[i], 0) @mx.operator.register('BoxAnnotatorOHEM') class BoxAnnotatorOHEMProp(mx.operator.CustomOpProp): def __init__(self, num_classes, num_reg_classes, roi_per_img): super(BoxAnnotatorOHEMProp, self).__init__(need_top_grad=False) self._num_classes = int(num_classes) self._num_reg_classes = int(num_reg_classes) self._roi_per_img = int(roi_per_img) def list_arguments(self): return ['cls_score', 'bbox_pred', 'labels', 'bbox_targets', 'bbox_weights'] def list_outputs(self): return ['labels_ohem', 'bbox_weights_ohem'] def infer_shape(self, in_shape): labels_shape = in_shape[2] bbox_weights_shape = in_shape[4] return in_shape, \ [labels_shape, bbox_weights_shape] def create_operator(self, ctx, shapes, dtypes): return BoxAnnotatorOHEMOperator(self._num_classes, self._num_reg_classes, self._roi_per_img) def declare_backward_dependency(self, out_grad, in_data, out_data): return [] ================================================ FILE: dff_rfcn/operator_py/proposal.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- """ Proposal Operator transform anchor coordinates into ROI coordinates with prediction results on classification probability and bounding box prediction results, and image size and scale information. """ import mxnet as mx import numpy as np import numpy.random as npr from distutils.util import strtobool from bbox.bbox_transform import bbox_pred, clip_boxes from rpn.generate_anchor import generate_anchors from nms.nms import py_nms_wrapper, cpu_nms_wrapper, gpu_nms_wrapper DEBUG = False class ProposalOperator(mx.operator.CustomOp): def __init__(self, feat_stride, scales, ratios, output_score, rpn_pre_nms_top_n, rpn_post_nms_top_n, threshold, rpn_min_size): super(ProposalOperator, self).__init__() self._feat_stride = feat_stride self._scales = np.fromstring(scales[1:-1], dtype=float, sep=',') self._ratios = np.fromstring(ratios[1:-1], dtype=float, sep=',') self._anchors = generate_anchors(base_size=self._feat_stride, scales=self._scales, ratios=self._ratios) self._num_anchors = self._anchors.shape[0] self._output_score = output_score self._rpn_pre_nms_top_n = rpn_pre_nms_top_n self._rpn_post_nms_top_n = rpn_post_nms_top_n self._threshold = threshold self._rpn_min_size = rpn_min_size if DEBUG: print 'feat_stride: {}'.format(self._feat_stride) print 'anchors:' print self._anchors def forward(self, is_train, req, in_data, out_data, aux): nms = gpu_nms_wrapper(self._threshold, in_data[0].context.device_id) batch_size = in_data[0].shape[0] if batch_size > 1: raise ValueError("Sorry, multiple images each device is not implemented") # for each (H, W) location i # generate A anchor boxes centered on cell i # apply predicted bbox deltas at cell i to each of the A anchors # clip predicted boxes to image # remove predicted boxes with either height or width < threshold # sort all (proposal, score) pairs by score from highest to lowest # take top pre_nms_topN proposals before NMS # apply NMS with threshold 0.7 to remaining proposals # take after_nms_topN proposals after NMS # return the top proposals (-> RoIs top, scores top) pre_nms_topN = self._rpn_pre_nms_top_n post_nms_topN = self._rpn_post_nms_top_n min_size = self._rpn_min_size # the first set of anchors are background probabilities # keep the second part scores = in_data[0].asnumpy()[:, self._num_anchors:, :, :] bbox_deltas = in_data[1].asnumpy() im_info = in_data[2].asnumpy()[0, :] if DEBUG: print 'im_size: ({}, {})'.format(im_info[0], im_info[1]) print 'scale: {}'.format(im_info[2]) # 1. Generate proposals from bbox_deltas and shifted anchors # use real image size instead of padded feature map sizes height, width = int(im_info[0] / self._feat_stride), int(im_info[1] / self._feat_stride) if DEBUG: print 'score map size: {}'.format(scores.shape) print "resudial: {}".format((scores.shape[2] - height, scores.shape[3] - width)) # Enumerate all shifts shift_x = np.arange(0, width) * self._feat_stride shift_y = np.arange(0, height) * self._feat_stride shift_x, shift_y = np.meshgrid(shift_x, shift_y) shifts = np.vstack((shift_x.ravel(), shift_y.ravel(), shift_x.ravel(), shift_y.ravel())).transpose() # Enumerate all shifted anchors: # # add A anchors (1, A, 4) to # cell K shifts (K, 1, 4) to get # shift anchors (K, A, 4) # reshape to (K*A, 4) shifted anchors A = self._num_anchors K = shifts.shape[0] anchors = self._anchors.reshape((1, A, 4)) + shifts.reshape((1, K, 4)).transpose((1, 0, 2)) anchors = anchors.reshape((K * A, 4)) # Transpose and reshape predicted bbox transformations to get them # into the same order as the anchors: # # bbox deltas will be (1, 4 * A, H, W) format # transpose to (1, H, W, 4 * A) # reshape to (1 * H * W * A, 4) where rows are ordered by (h, w, a) # in slowest to fastest order bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4)) # Same story for the scores: # # scores are (1, A, H, W) format # transpose to (1, H, W, A) # reshape to (1 * H * W * A, 1) where rows are ordered by (h, w, a) scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) # Convert anchors into proposals via bbox transformations proposals = bbox_pred(anchors, bbox_deltas) # 2. clip predicted boxes to image proposals = clip_boxes(proposals, im_info[:2]) # 3. remove predicted boxes with either height or width < threshold # (NOTE: convert min_size to input image scale stored in im_info[2]) keep = self._filter_boxes(proposals, min_size * im_info[2]) proposals = proposals[keep, :] scores = scores[keep] # 4. sort all (proposal, score) pairs by score from highest to lowest # 5. take top pre_nms_topN (e.g. 6000) order = scores.ravel().argsort()[::-1] if pre_nms_topN > 0: order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] # 6. apply nms (e.g. threshold = 0.7) # 7. take after_nms_topN (e.g. 300) # 8. return the top proposals (-> RoIs top) det = np.hstack((proposals, scores)).astype(np.float32) keep = nms(det) if post_nms_topN > 0: keep = keep[:post_nms_topN] # pad to ensure output size remains unchanged if len(keep) < post_nms_topN: pad = npr.choice(keep, size=post_nms_topN - len(keep)) keep = np.hstack((keep, pad)) proposals = proposals[keep, :] scores = scores[keep] # Output rois array # Our RPN implementation only supports a single input image, so all # batch inds are 0 batch_inds = np.zeros((proposals.shape[0], 1), dtype=np.float32) blob = np.hstack((batch_inds, proposals.astype(np.float32, copy=False))) self.assign(out_data[0], req[0], blob) if self._output_score: self.assign(out_data[1], req[1], scores.astype(np.float32, copy=False)) def backward(self, req, out_grad, in_data, out_data, in_grad, aux): self.assign(in_grad[0], req[0], 0) self.assign(in_grad[1], req[1], 0) self.assign(in_grad[2], req[2], 0) @staticmethod def _filter_boxes(boxes, min_size): """ Remove all boxes with any side smaller than min_size """ ws = boxes[:, 2] - boxes[:, 0] + 1 hs = boxes[:, 3] - boxes[:, 1] + 1 keep = np.where((ws >= min_size) & (hs >= min_size))[0] return keep @staticmethod def _clip_pad(tensor, pad_shape): """ Clip boxes of the pad area. :param tensor: [n, c, H, W] :param pad_shape: [h, w] :return: [n, c, h, w] """ H, W = tensor.shape[2:] h, w = pad_shape if h < H or w < W: tensor = tensor[:, :, :h, :w].copy() return tensor @mx.operator.register("proposal") class ProposalProp(mx.operator.CustomOpProp): def __init__(self, feat_stride='16', scales='(8, 16, 32)', ratios='(0.5, 1, 2)', output_score='False', rpn_pre_nms_top_n='6000', rpn_post_nms_top_n='300', threshold='0.3', rpn_min_size='16'): super(ProposalProp, self).__init__(need_top_grad=False) self._feat_stride = int(feat_stride) self._scales = scales self._ratios = ratios self._output_score = strtobool(output_score) self._rpn_pre_nms_top_n = int(rpn_pre_nms_top_n) self._rpn_post_nms_top_n = int(rpn_post_nms_top_n) self._threshold = float(threshold) self._rpn_min_size = int(rpn_min_size) def list_arguments(self): return ['cls_prob', 'bbox_pred', 'im_info'] def list_outputs(self): if self._output_score: return ['output', 'score'] else: return ['output'] def infer_shape(self, in_shape): cls_prob_shape = in_shape[0] bbox_pred_shape = in_shape[1] assert cls_prob_shape[0] == bbox_pred_shape[0], 'ROI number does not equal in cls and reg' batch_size = cls_prob_shape[0] im_info_shape = (batch_size, 3) output_shape = (self._rpn_post_nms_top_n, 5) score_shape = (self._rpn_post_nms_top_n, 1) if self._output_score: return [cls_prob_shape, bbox_pred_shape, im_info_shape], [output_shape, score_shape] else: return [cls_prob_shape, bbox_pred_shape, im_info_shape], [output_shape] def create_operator(self, ctx, shapes, dtypes): return ProposalOperator(self._feat_stride, self._scales, self._ratios, self._output_score, self._rpn_pre_nms_top_n, self._rpn_post_nms_top_n, self._threshold, self._rpn_min_size) def declare_backward_dependency(self, out_grad, in_data, out_data): return [] ================================================ FILE: dff_rfcn/operator_py/proposal_target.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- """ Proposal Target Operator selects foreground and background roi and assigns label, bbox_transform to them. """ import mxnet as mx import numpy as np from distutils.util import strtobool from easydict import EasyDict as edict import cPickle from core.rcnn import sample_rois DEBUG = False class ProposalTargetOperator(mx.operator.CustomOp): def __init__(self, num_classes, batch_images, batch_rois, cfg, fg_fraction): super(ProposalTargetOperator, self).__init__() self._num_classes = num_classes self._batch_images = batch_images self._batch_rois = batch_rois self._cfg = cfg self._fg_fraction = fg_fraction if DEBUG: self._count = 0 self._fg_num = 0 self._bg_num = 0 def forward(self, is_train, req, in_data, out_data, aux): assert self._batch_rois == -1 or self._batch_rois % self._batch_images == 0, \ 'batchimages {} must devide batch_rois {}'.format(self._batch_images, self._batch_rois) all_rois = in_data[0].asnumpy() gt_boxes = in_data[1].asnumpy() if self._batch_rois == -1: rois_per_image = all_rois.shape[0] + gt_boxes.shape[0] fg_rois_per_image = rois_per_image else: rois_per_image = self._batch_rois / self._batch_images fg_rois_per_image = np.round(self._fg_fraction * rois_per_image).astype(int) # Include ground-truth boxes in the set of candidate rois zeros = np.zeros((gt_boxes.shape[0], 1), dtype=gt_boxes.dtype) all_rois = np.vstack((all_rois, np.hstack((zeros, gt_boxes[:, :-1])))) # Sanity check: single batch only assert np.all(all_rois[:, 0] == 0), 'Only single item batches are supported' rois, labels, bbox_targets, bbox_weights = \ sample_rois(all_rois, fg_rois_per_image, rois_per_image, self._num_classes, self._cfg, gt_boxes=gt_boxes) if DEBUG: print "labels=", labels print 'num fg: {}'.format((labels > 0).sum()) print 'num bg: {}'.format((labels == 0).sum()) self._count += 1 self._fg_num += (labels > 0).sum() self._bg_num += (labels == 0).sum() print "self._count=", self._count print 'num fg avg: {}'.format(self._fg_num / self._count) print 'num bg avg: {}'.format(self._bg_num / self._count) print 'ratio: {:.3f}'.format(float(self._fg_num) / float(self._bg_num)) for ind, val in enumerate([rois, labels, bbox_targets, bbox_weights]): self.assign(out_data[ind], req[ind], val) def backward(self, req, out_grad, in_data, out_data, in_grad, aux): self.assign(in_grad[0], req[0], 0) self.assign(in_grad[1], req[1], 0) @mx.operator.register('proposal_target') class ProposalTargetProp(mx.operator.CustomOpProp): def __init__(self, num_classes, batch_images, batch_rois, cfg, fg_fraction='0.25'): super(ProposalTargetProp, self).__init__(need_top_grad=False) self._num_classes = int(num_classes) self._batch_images = int(batch_images) self._batch_rois = int(batch_rois) self._cfg = cPickle.loads(cfg) self._fg_fraction = float(fg_fraction) def list_arguments(self): return ['rois', 'gt_boxes'] def list_outputs(self): return ['rois_output', 'label', 'bbox_target', 'bbox_weight'] def infer_shape(self, in_shape): rpn_rois_shape = in_shape[0] gt_boxes_shape = in_shape[1] rois = rpn_rois_shape[0] + gt_boxes_shape[0] if self._batch_rois == -1 else self._batch_rois output_rois_shape = (rois, 5) label_shape = (rois, ) bbox_target_shape = (rois, self._num_classes * 4) bbox_weight_shape = (rois, self._num_classes * 4) return [rpn_rois_shape, gt_boxes_shape], \ [output_rois_shape, label_shape, bbox_target_shape, bbox_weight_shape] def create_operator(self, ctx, shapes, dtypes): return ProposalTargetOperator(self._num_classes, self._batch_images, self._batch_rois, self._cfg, self._fg_fraction) def declare_backward_dependency(self, out_grad, in_data, out_data): return [] ================================================ FILE: dff_rfcn/operator_py/rpn_inv_normalize.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Xizhou Zhu # -------------------------------------------------------- import mxnet as mx import numpy as np from distutils.util import strtobool class RPNInvNormalizeOperator(mx.operator.CustomOp): def __init__(self, num_anchors, bbox_mean, bbox_std): super(RPNInvNormalizeOperator, self).__init__() self._num_anchors = num_anchors self._bbox_mean = mx.ndarray.Reshape(mx.nd.array(bbox_mean), shape=(1,4,1,1)) self._bbox_std = mx.ndarray.Reshape(mx.nd.array(bbox_std), shape=(1,4,1,1)) def forward(self, is_train, req, in_data, out_data, aux): bbox_pred = in_data[0] tile_shape = (bbox_pred.shape[0], self._num_anchors, bbox_pred.shape[2], bbox_pred.shape[3]) bbox_mean = mx.ndarray.tile(self._bbox_mean.as_in_context(bbox_pred.context), reps=tile_shape) bbox_std = mx.ndarray.tile(self._bbox_std.as_in_context(bbox_pred.context), reps=tile_shape) bbox_pred = bbox_pred * bbox_std + bbox_mean self.assign(out_data[0], req[0], bbox_pred) def backward(self, req, out_grad, in_data, out_data, in_grad, aux): self.assign(in_grad[0], req[0], 0) @mx.operator.register('rpn_inv_normalize') class RPNInvNormalizeProp(mx.operator.CustomOpProp): def __init__(self, num_anchors, bbox_mean='(0.0, 0.0, 0.0, 0.0)', bbox_std='0.1, 0.1, 0.2, 0.2'): super(RPNInvNormalizeProp, self).__init__(need_top_grad=False) self._num_anchors = int(num_anchors) self._bbox_mean = np.fromstring(bbox_mean[1:-1], dtype=float, sep=',') self._bbox_std = np.fromstring(bbox_std[1:-1], dtype=float, sep=',') def list_arguments(self): return ['bbox_pred'] def list_outputs(self): return ['out_bbox_pred'] def infer_shape(self, in_shape): return [in_shape[0]], \ [in_shape[0]] def create_operator(self, ctx, shapes, dtypes): return RPNInvNormalizeOperator(self._num_anchors, self._bbox_mean, self._bbox_std) def declare_backward_dependency(self, out_grad, in_data, out_data): return [] ================================================ FILE: dff_rfcn/operator_py/tile_as.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Xizhou Zhu # -------------------------------------------------------- import mxnet as mx import numpy as np from distutils.util import strtobool class TileAsOperator(mx.operator.CustomOp): def __init__(self): super(TileAsOperator, self).__init__() def forward(self, is_train, req, in_data, out_data, aux): data_content = in_data[0] data_tiled = mx.ndarray.tile(data_content, reps=(in_data[1].shape[0], 1, 1, 1)) self.assign(out_data[0], req[0], data_tiled) def backward(self, req, out_grad, in_data, out_data, in_grad, aux): self.assign(in_grad[0], req[0], 0) self.assign(in_grad[1], req[1], 0) @mx.operator.register('tile_as') class TileAsProp(mx.operator.CustomOpProp): def __init__(self): super(TileAsProp, self).__init__(need_top_grad=False) def list_arguments(self): return ['data_content', 'data_shape'] def list_outputs(self): return ['data_tiled'] def infer_shape(self, in_shape): data_content_shape = in_shape[0] data_shape_shape = in_shape[1] tiled_data_shape = (data_shape_shape[0], data_content_shape[1], data_content_shape[2], data_content_shape[3]) return [data_content_shape, data_shape_shape], \ [tiled_data_shape] def create_operator(self, ctx, shapes, dtypes): return TileAsOperator() def declare_backward_dependency(self, out_grad, in_data, out_data): return out_grad ================================================ FILE: dff_rfcn/symbols/__init__.py ================================================ import resnet_v1_101_flownet_rfcn ================================================ FILE: dff_rfcn/symbols/resnet_v1_101_flownet_rfcn.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong, Xizhou Zhu # -------------------------------------------------------- import cPickle import mxnet as mx from utils.symbol import Symbol from operator_py.proposal import * from operator_py.proposal_target import * from operator_py.box_annotator_ohem import * from operator_py.rpn_inv_normalize import * from operator_py.tile_as import * class resnet_v1_101_flownet_rfcn(Symbol): def __init__(self): """ Use __init__ to define parameter network needs """ self.eps = 1e-5 self.use_global_stats = True self.workspace = 512 self.units = (3, 4, 23, 3) # use for 101 self.filter_list = [256, 512, 1024, 2048] def get_resnet_v1(self, data): conv1 = mx.symbol.Convolution(name='conv1', data=data , num_filter=64, pad=(3,3), kernel=(7,7), stride=(2,2), no_bias=True) bn_conv1 = mx.symbol.BatchNorm(name='bn_conv1', data=conv1 , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale_conv1 = bn_conv1 conv1_relu = mx.symbol.Activation(name='conv1_relu', data=scale_conv1 , act_type='relu') pool1 = mx.symbol.Pooling(name='pool1', data=conv1_relu , pad=(1,1), kernel=(3,3), stride=(2,2), pool_type='max') res2a_branch1 = mx.symbol.Convolution(name='res2a_branch1', data=pool1 , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn2a_branch1 = mx.symbol.BatchNorm(name='bn2a_branch1', data=res2a_branch1 , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale2a_branch1 = bn2a_branch1 res2a_branch2a = mx.symbol.Convolution(name='res2a_branch2a', data=pool1 , num_filter=64, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn2a_branch2a = mx.symbol.BatchNorm(name='bn2a_branch2a', data=res2a_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale2a_branch2a = bn2a_branch2a res2a_branch2a_relu = mx.symbol.Activation(name='res2a_branch2a_relu', data=scale2a_branch2a , act_type='relu') res2a_branch2b = mx.symbol.Convolution(name='res2a_branch2b', data=res2a_branch2a_relu , num_filter=64, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn2a_branch2b = mx.symbol.BatchNorm(name='bn2a_branch2b', data=res2a_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale2a_branch2b = bn2a_branch2b res2a_branch2b_relu = mx.symbol.Activation(name='res2a_branch2b_relu', data=scale2a_branch2b , act_type='relu') res2a_branch2c = mx.symbol.Convolution(name='res2a_branch2c', data=res2a_branch2b_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn2a_branch2c = mx.symbol.BatchNorm(name='bn2a_branch2c', data=res2a_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale2a_branch2c = bn2a_branch2c res2a = mx.symbol.broadcast_add(name='res2a', *[scale2a_branch1,scale2a_branch2c] ) res2a_relu = mx.symbol.Activation(name='res2a_relu', data=res2a , act_type='relu') res2b_branch2a = mx.symbol.Convolution(name='res2b_branch2a', data=res2a_relu , num_filter=64, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn2b_branch2a = mx.symbol.BatchNorm(name='bn2b_branch2a', data=res2b_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale2b_branch2a = bn2b_branch2a res2b_branch2a_relu = mx.symbol.Activation(name='res2b_branch2a_relu', data=scale2b_branch2a , act_type='relu') res2b_branch2b = mx.symbol.Convolution(name='res2b_branch2b', data=res2b_branch2a_relu , num_filter=64, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn2b_branch2b = mx.symbol.BatchNorm(name='bn2b_branch2b', data=res2b_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale2b_branch2b = bn2b_branch2b res2b_branch2b_relu = mx.symbol.Activation(name='res2b_branch2b_relu', data=scale2b_branch2b , act_type='relu') res2b_branch2c = mx.symbol.Convolution(name='res2b_branch2c', data=res2b_branch2b_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn2b_branch2c = mx.symbol.BatchNorm(name='bn2b_branch2c', data=res2b_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale2b_branch2c = bn2b_branch2c res2b = mx.symbol.broadcast_add(name='res2b', *[res2a_relu,scale2b_branch2c] ) res2b_relu = mx.symbol.Activation(name='res2b_relu', data=res2b , act_type='relu') res2c_branch2a = mx.symbol.Convolution(name='res2c_branch2a', data=res2b_relu , num_filter=64, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn2c_branch2a = mx.symbol.BatchNorm(name='bn2c_branch2a', data=res2c_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale2c_branch2a = bn2c_branch2a res2c_branch2a_relu = mx.symbol.Activation(name='res2c_branch2a_relu', data=scale2c_branch2a , act_type='relu') res2c_branch2b = mx.symbol.Convolution(name='res2c_branch2b', data=res2c_branch2a_relu , num_filter=64, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn2c_branch2b = mx.symbol.BatchNorm(name='bn2c_branch2b', data=res2c_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale2c_branch2b = bn2c_branch2b res2c_branch2b_relu = mx.symbol.Activation(name='res2c_branch2b_relu', data=scale2c_branch2b , act_type='relu') res2c_branch2c = mx.symbol.Convolution(name='res2c_branch2c', data=res2c_branch2b_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn2c_branch2c = mx.symbol.BatchNorm(name='bn2c_branch2c', data=res2c_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale2c_branch2c = bn2c_branch2c res2c = mx.symbol.broadcast_add(name='res2c', *[res2b_relu,scale2c_branch2c] ) res2c_relu = mx.symbol.Activation(name='res2c_relu', data=res2c , act_type='relu') res3a_branch1 = mx.symbol.Convolution(name='res3a_branch1', data=res2c_relu , num_filter=512, pad=(0,0), kernel=(1,1), stride=(2,2), no_bias=True) bn3a_branch1 = mx.symbol.BatchNorm(name='bn3a_branch1', data=res3a_branch1 , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3a_branch1 = bn3a_branch1 res3a_branch2a = mx.symbol.Convolution(name='res3a_branch2a', data=res2c_relu , num_filter=128, pad=(0,0), kernel=(1,1), stride=(2,2), no_bias=True) bn3a_branch2a = mx.symbol.BatchNorm(name='bn3a_branch2a', data=res3a_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3a_branch2a = bn3a_branch2a res3a_branch2a_relu = mx.symbol.Activation(name='res3a_branch2a_relu', data=scale3a_branch2a , act_type='relu') res3a_branch2b = mx.symbol.Convolution(name='res3a_branch2b', data=res3a_branch2a_relu , num_filter=128, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn3a_branch2b = mx.symbol.BatchNorm(name='bn3a_branch2b', data=res3a_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3a_branch2b = bn3a_branch2b res3a_branch2b_relu = mx.symbol.Activation(name='res3a_branch2b_relu', data=scale3a_branch2b , act_type='relu') res3a_branch2c = mx.symbol.Convolution(name='res3a_branch2c', data=res3a_branch2b_relu , num_filter=512, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn3a_branch2c = mx.symbol.BatchNorm(name='bn3a_branch2c', data=res3a_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3a_branch2c = bn3a_branch2c res3a = mx.symbol.broadcast_add(name='res3a', *[scale3a_branch1,scale3a_branch2c] ) res3a_relu = mx.symbol.Activation(name='res3a_relu', data=res3a , act_type='relu') res3b1_branch2a = mx.symbol.Convolution(name='res3b1_branch2a', data=res3a_relu , num_filter=128, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn3b1_branch2a = mx.symbol.BatchNorm(name='bn3b1_branch2a', data=res3b1_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3b1_branch2a = bn3b1_branch2a res3b1_branch2a_relu = mx.symbol.Activation(name='res3b1_branch2a_relu', data=scale3b1_branch2a , act_type='relu') res3b1_branch2b = mx.symbol.Convolution(name='res3b1_branch2b', data=res3b1_branch2a_relu , num_filter=128, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn3b1_branch2b = mx.symbol.BatchNorm(name='bn3b1_branch2b', data=res3b1_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3b1_branch2b = bn3b1_branch2b res3b1_branch2b_relu = mx.symbol.Activation(name='res3b1_branch2b_relu', data=scale3b1_branch2b , act_type='relu') res3b1_branch2c = mx.symbol.Convolution(name='res3b1_branch2c', data=res3b1_branch2b_relu , num_filter=512, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn3b1_branch2c = mx.symbol.BatchNorm(name='bn3b1_branch2c', data=res3b1_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3b1_branch2c = bn3b1_branch2c res3b1 = mx.symbol.broadcast_add(name='res3b1', *[res3a_relu,scale3b1_branch2c] ) res3b1_relu = mx.symbol.Activation(name='res3b1_relu', data=res3b1 , act_type='relu') res3b2_branch2a = mx.symbol.Convolution(name='res3b2_branch2a', data=res3b1_relu , num_filter=128, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn3b2_branch2a = mx.symbol.BatchNorm(name='bn3b2_branch2a', data=res3b2_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3b2_branch2a = bn3b2_branch2a res3b2_branch2a_relu = mx.symbol.Activation(name='res3b2_branch2a_relu', data=scale3b2_branch2a , act_type='relu') res3b2_branch2b = mx.symbol.Convolution(name='res3b2_branch2b', data=res3b2_branch2a_relu , num_filter=128, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn3b2_branch2b = mx.symbol.BatchNorm(name='bn3b2_branch2b', data=res3b2_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3b2_branch2b = bn3b2_branch2b res3b2_branch2b_relu = mx.symbol.Activation(name='res3b2_branch2b_relu', data=scale3b2_branch2b , act_type='relu') res3b2_branch2c = mx.symbol.Convolution(name='res3b2_branch2c', data=res3b2_branch2b_relu , num_filter=512, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn3b2_branch2c = mx.symbol.BatchNorm(name='bn3b2_branch2c', data=res3b2_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3b2_branch2c = bn3b2_branch2c res3b2 = mx.symbol.broadcast_add(name='res3b2', *[res3b1_relu,scale3b2_branch2c] ) res3b2_relu = mx.symbol.Activation(name='res3b2_relu', data=res3b2 , act_type='relu') res3b3_branch2a = mx.symbol.Convolution(name='res3b3_branch2a', data=res3b2_relu , num_filter=128, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn3b3_branch2a = mx.symbol.BatchNorm(name='bn3b3_branch2a', data=res3b3_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3b3_branch2a = bn3b3_branch2a res3b3_branch2a_relu = mx.symbol.Activation(name='res3b3_branch2a_relu', data=scale3b3_branch2a , act_type='relu') res3b3_branch2b = mx.symbol.Convolution(name='res3b3_branch2b', data=res3b3_branch2a_relu , num_filter=128, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn3b3_branch2b = mx.symbol.BatchNorm(name='bn3b3_branch2b', data=res3b3_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3b3_branch2b = bn3b3_branch2b res3b3_branch2b_relu = mx.symbol.Activation(name='res3b3_branch2b_relu', data=scale3b3_branch2b , act_type='relu') res3b3_branch2c = mx.symbol.Convolution(name='res3b3_branch2c', data=res3b3_branch2b_relu , num_filter=512, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn3b3_branch2c = mx.symbol.BatchNorm(name='bn3b3_branch2c', data=res3b3_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3b3_branch2c = bn3b3_branch2c res3b3 = mx.symbol.broadcast_add(name='res3b3', *[res3b2_relu,scale3b3_branch2c] ) res3b3_relu = mx.symbol.Activation(name='res3b3_relu', data=res3b3 , act_type='relu') res4a_branch1 = mx.symbol.Convolution(name='res4a_branch1', data=res3b3_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(2,2), no_bias=True) bn4a_branch1 = mx.symbol.BatchNorm(name='bn4a_branch1', data=res4a_branch1 , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4a_branch1 = bn4a_branch1 res4a_branch2a = mx.symbol.Convolution(name='res4a_branch2a', data=res3b3_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(2,2), no_bias=True) bn4a_branch2a = mx.symbol.BatchNorm(name='bn4a_branch2a', data=res4a_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4a_branch2a = bn4a_branch2a res4a_branch2a_relu = mx.symbol.Activation(name='res4a_branch2a_relu', data=scale4a_branch2a , act_type='relu') res4a_branch2b = mx.symbol.Convolution(name='res4a_branch2b', data=res4a_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4a_branch2b = mx.symbol.BatchNorm(name='bn4a_branch2b', data=res4a_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4a_branch2b = bn4a_branch2b res4a_branch2b_relu = mx.symbol.Activation(name='res4a_branch2b_relu', data=scale4a_branch2b , act_type='relu') res4a_branch2c = mx.symbol.Convolution(name='res4a_branch2c', data=res4a_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4a_branch2c = mx.symbol.BatchNorm(name='bn4a_branch2c', data=res4a_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4a_branch2c = bn4a_branch2c res4a = mx.symbol.broadcast_add(name='res4a', *[scale4a_branch1,scale4a_branch2c] ) res4a_relu = mx.symbol.Activation(name='res4a_relu', data=res4a , act_type='relu') res4b1_branch2a = mx.symbol.Convolution(name='res4b1_branch2a', data=res4a_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b1_branch2a = mx.symbol.BatchNorm(name='bn4b1_branch2a', data=res4b1_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b1_branch2a = bn4b1_branch2a res4b1_branch2a_relu = mx.symbol.Activation(name='res4b1_branch2a_relu', data=scale4b1_branch2a , act_type='relu') res4b1_branch2b = mx.symbol.Convolution(name='res4b1_branch2b', data=res4b1_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b1_branch2b = mx.symbol.BatchNorm(name='bn4b1_branch2b', data=res4b1_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b1_branch2b = bn4b1_branch2b res4b1_branch2b_relu = mx.symbol.Activation(name='res4b1_branch2b_relu', data=scale4b1_branch2b , act_type='relu') res4b1_branch2c = mx.symbol.Convolution(name='res4b1_branch2c', data=res4b1_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b1_branch2c = mx.symbol.BatchNorm(name='bn4b1_branch2c', data=res4b1_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b1_branch2c = bn4b1_branch2c res4b1 = mx.symbol.broadcast_add(name='res4b1', *[res4a_relu,scale4b1_branch2c] ) res4b1_relu = mx.symbol.Activation(name='res4b1_relu', data=res4b1 , act_type='relu') res4b2_branch2a = mx.symbol.Convolution(name='res4b2_branch2a', data=res4b1_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b2_branch2a = mx.symbol.BatchNorm(name='bn4b2_branch2a', data=res4b2_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b2_branch2a = bn4b2_branch2a res4b2_branch2a_relu = mx.symbol.Activation(name='res4b2_branch2a_relu', data=scale4b2_branch2a , act_type='relu') res4b2_branch2b = mx.symbol.Convolution(name='res4b2_branch2b', data=res4b2_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b2_branch2b = mx.symbol.BatchNorm(name='bn4b2_branch2b', data=res4b2_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b2_branch2b = bn4b2_branch2b res4b2_branch2b_relu = mx.symbol.Activation(name='res4b2_branch2b_relu', data=scale4b2_branch2b , act_type='relu') res4b2_branch2c = mx.symbol.Convolution(name='res4b2_branch2c', data=res4b2_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b2_branch2c = mx.symbol.BatchNorm(name='bn4b2_branch2c', data=res4b2_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b2_branch2c = bn4b2_branch2c res4b2 = mx.symbol.broadcast_add(name='res4b2', *[res4b1_relu,scale4b2_branch2c] ) res4b2_relu = mx.symbol.Activation(name='res4b2_relu', data=res4b2 , act_type='relu') res4b3_branch2a = mx.symbol.Convolution(name='res4b3_branch2a', data=res4b2_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b3_branch2a = mx.symbol.BatchNorm(name='bn4b3_branch2a', data=res4b3_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b3_branch2a = bn4b3_branch2a res4b3_branch2a_relu = mx.symbol.Activation(name='res4b3_branch2a_relu', data=scale4b3_branch2a , act_type='relu') res4b3_branch2b = mx.symbol.Convolution(name='res4b3_branch2b', data=res4b3_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b3_branch2b = mx.symbol.BatchNorm(name='bn4b3_branch2b', data=res4b3_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b3_branch2b = bn4b3_branch2b res4b3_branch2b_relu = mx.symbol.Activation(name='res4b3_branch2b_relu', data=scale4b3_branch2b , act_type='relu') res4b3_branch2c = mx.symbol.Convolution(name='res4b3_branch2c', data=res4b3_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b3_branch2c = mx.symbol.BatchNorm(name='bn4b3_branch2c', data=res4b3_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b3_branch2c = bn4b3_branch2c res4b3 = mx.symbol.broadcast_add(name='res4b3', *[res4b2_relu,scale4b3_branch2c] ) res4b3_relu = mx.symbol.Activation(name='res4b3_relu', data=res4b3 , act_type='relu') res4b4_branch2a = mx.symbol.Convolution(name='res4b4_branch2a', data=res4b3_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b4_branch2a = mx.symbol.BatchNorm(name='bn4b4_branch2a', data=res4b4_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b4_branch2a = bn4b4_branch2a res4b4_branch2a_relu = mx.symbol.Activation(name='res4b4_branch2a_relu', data=scale4b4_branch2a , act_type='relu') res4b4_branch2b = mx.symbol.Convolution(name='res4b4_branch2b', data=res4b4_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b4_branch2b = mx.symbol.BatchNorm(name='bn4b4_branch2b', data=res4b4_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b4_branch2b = bn4b4_branch2b res4b4_branch2b_relu = mx.symbol.Activation(name='res4b4_branch2b_relu', data=scale4b4_branch2b , act_type='relu') res4b4_branch2c = mx.symbol.Convolution(name='res4b4_branch2c', data=res4b4_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b4_branch2c = mx.symbol.BatchNorm(name='bn4b4_branch2c', data=res4b4_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b4_branch2c = bn4b4_branch2c res4b4 = mx.symbol.broadcast_add(name='res4b4', *[res4b3_relu,scale4b4_branch2c] ) res4b4_relu = mx.symbol.Activation(name='res4b4_relu', data=res4b4 , act_type='relu') res4b5_branch2a = mx.symbol.Convolution(name='res4b5_branch2a', data=res4b4_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b5_branch2a = mx.symbol.BatchNorm(name='bn4b5_branch2a', data=res4b5_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b5_branch2a = bn4b5_branch2a res4b5_branch2a_relu = mx.symbol.Activation(name='res4b5_branch2a_relu', data=scale4b5_branch2a , act_type='relu') res4b5_branch2b = mx.symbol.Convolution(name='res4b5_branch2b', data=res4b5_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b5_branch2b = mx.symbol.BatchNorm(name='bn4b5_branch2b', data=res4b5_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b5_branch2b = bn4b5_branch2b res4b5_branch2b_relu = mx.symbol.Activation(name='res4b5_branch2b_relu', data=scale4b5_branch2b , act_type='relu') res4b5_branch2c = mx.symbol.Convolution(name='res4b5_branch2c', data=res4b5_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b5_branch2c = mx.symbol.BatchNorm(name='bn4b5_branch2c', data=res4b5_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b5_branch2c = bn4b5_branch2c res4b5 = mx.symbol.broadcast_add(name='res4b5', *[res4b4_relu,scale4b5_branch2c] ) res4b5_relu = mx.symbol.Activation(name='res4b5_relu', data=res4b5 , act_type='relu') res4b6_branch2a = mx.symbol.Convolution(name='res4b6_branch2a', data=res4b5_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b6_branch2a = mx.symbol.BatchNorm(name='bn4b6_branch2a', data=res4b6_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b6_branch2a = bn4b6_branch2a res4b6_branch2a_relu = mx.symbol.Activation(name='res4b6_branch2a_relu', data=scale4b6_branch2a , act_type='relu') res4b6_branch2b = mx.symbol.Convolution(name='res4b6_branch2b', data=res4b6_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b6_branch2b = mx.symbol.BatchNorm(name='bn4b6_branch2b', data=res4b6_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b6_branch2b = bn4b6_branch2b res4b6_branch2b_relu = mx.symbol.Activation(name='res4b6_branch2b_relu', data=scale4b6_branch2b , act_type='relu') res4b6_branch2c = mx.symbol.Convolution(name='res4b6_branch2c', data=res4b6_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b6_branch2c = mx.symbol.BatchNorm(name='bn4b6_branch2c', data=res4b6_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b6_branch2c = bn4b6_branch2c res4b6 = mx.symbol.broadcast_add(name='res4b6', *[res4b5_relu,scale4b6_branch2c] ) res4b6_relu = mx.symbol.Activation(name='res4b6_relu', data=res4b6 , act_type='relu') res4b7_branch2a = mx.symbol.Convolution(name='res4b7_branch2a', data=res4b6_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b7_branch2a = mx.symbol.BatchNorm(name='bn4b7_branch2a', data=res4b7_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b7_branch2a = bn4b7_branch2a res4b7_branch2a_relu = mx.symbol.Activation(name='res4b7_branch2a_relu', data=scale4b7_branch2a , act_type='relu') res4b7_branch2b = mx.symbol.Convolution(name='res4b7_branch2b', data=res4b7_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b7_branch2b = mx.symbol.BatchNorm(name='bn4b7_branch2b', data=res4b7_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b7_branch2b = bn4b7_branch2b res4b7_branch2b_relu = mx.symbol.Activation(name='res4b7_branch2b_relu', data=scale4b7_branch2b , act_type='relu') res4b7_branch2c = mx.symbol.Convolution(name='res4b7_branch2c', data=res4b7_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b7_branch2c = mx.symbol.BatchNorm(name='bn4b7_branch2c', data=res4b7_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b7_branch2c = bn4b7_branch2c res4b7 = mx.symbol.broadcast_add(name='res4b7', *[res4b6_relu,scale4b7_branch2c] ) res4b7_relu = mx.symbol.Activation(name='res4b7_relu', data=res4b7 , act_type='relu') res4b8_branch2a = mx.symbol.Convolution(name='res4b8_branch2a', data=res4b7_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b8_branch2a = mx.symbol.BatchNorm(name='bn4b8_branch2a', data=res4b8_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b8_branch2a = bn4b8_branch2a res4b8_branch2a_relu = mx.symbol.Activation(name='res4b8_branch2a_relu', data=scale4b8_branch2a , act_type='relu') res4b8_branch2b = mx.symbol.Convolution(name='res4b8_branch2b', data=res4b8_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b8_branch2b = mx.symbol.BatchNorm(name='bn4b8_branch2b', data=res4b8_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b8_branch2b = bn4b8_branch2b res4b8_branch2b_relu = mx.symbol.Activation(name='res4b8_branch2b_relu', data=scale4b8_branch2b , act_type='relu') res4b8_branch2c = mx.symbol.Convolution(name='res4b8_branch2c', data=res4b8_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b8_branch2c = mx.symbol.BatchNorm(name='bn4b8_branch2c', data=res4b8_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b8_branch2c = bn4b8_branch2c res4b8 = mx.symbol.broadcast_add(name='res4b8', *[res4b7_relu,scale4b8_branch2c] ) res4b8_relu = mx.symbol.Activation(name='res4b8_relu', data=res4b8 , act_type='relu') res4b9_branch2a = mx.symbol.Convolution(name='res4b9_branch2a', data=res4b8_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b9_branch2a = mx.symbol.BatchNorm(name='bn4b9_branch2a', data=res4b9_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b9_branch2a = bn4b9_branch2a res4b9_branch2a_relu = mx.symbol.Activation(name='res4b9_branch2a_relu', data=scale4b9_branch2a , act_type='relu') res4b9_branch2b = mx.symbol.Convolution(name='res4b9_branch2b', data=res4b9_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b9_branch2b = mx.symbol.BatchNorm(name='bn4b9_branch2b', data=res4b9_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b9_branch2b = bn4b9_branch2b res4b9_branch2b_relu = mx.symbol.Activation(name='res4b9_branch2b_relu', data=scale4b9_branch2b , act_type='relu') res4b9_branch2c = mx.symbol.Convolution(name='res4b9_branch2c', data=res4b9_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b9_branch2c = mx.symbol.BatchNorm(name='bn4b9_branch2c', data=res4b9_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b9_branch2c = bn4b9_branch2c res4b9 = mx.symbol.broadcast_add(name='res4b9', *[res4b8_relu,scale4b9_branch2c] ) res4b9_relu = mx.symbol.Activation(name='res4b9_relu', data=res4b9 , act_type='relu') res4b10_branch2a = mx.symbol.Convolution(name='res4b10_branch2a', data=res4b9_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b10_branch2a = mx.symbol.BatchNorm(name='bn4b10_branch2a', data=res4b10_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b10_branch2a = bn4b10_branch2a res4b10_branch2a_relu = mx.symbol.Activation(name='res4b10_branch2a_relu', data=scale4b10_branch2a , act_type='relu') res4b10_branch2b = mx.symbol.Convolution(name='res4b10_branch2b', data=res4b10_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b10_branch2b = mx.symbol.BatchNorm(name='bn4b10_branch2b', data=res4b10_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b10_branch2b = bn4b10_branch2b res4b10_branch2b_relu = mx.symbol.Activation(name='res4b10_branch2b_relu', data=scale4b10_branch2b , act_type='relu') res4b10_branch2c = mx.symbol.Convolution(name='res4b10_branch2c', data=res4b10_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b10_branch2c = mx.symbol.BatchNorm(name='bn4b10_branch2c', data=res4b10_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b10_branch2c = bn4b10_branch2c res4b10 = mx.symbol.broadcast_add(name='res4b10', *[res4b9_relu,scale4b10_branch2c] ) res4b10_relu = mx.symbol.Activation(name='res4b10_relu', data=res4b10 , act_type='relu') res4b11_branch2a = mx.symbol.Convolution(name='res4b11_branch2a', data=res4b10_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b11_branch2a = mx.symbol.BatchNorm(name='bn4b11_branch2a', data=res4b11_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b11_branch2a = bn4b11_branch2a res4b11_branch2a_relu = mx.symbol.Activation(name='res4b11_branch2a_relu', data=scale4b11_branch2a , act_type='relu') res4b11_branch2b = mx.symbol.Convolution(name='res4b11_branch2b', data=res4b11_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b11_branch2b = mx.symbol.BatchNorm(name='bn4b11_branch2b', data=res4b11_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b11_branch2b = bn4b11_branch2b res4b11_branch2b_relu = mx.symbol.Activation(name='res4b11_branch2b_relu', data=scale4b11_branch2b , act_type='relu') res4b11_branch2c = mx.symbol.Convolution(name='res4b11_branch2c', data=res4b11_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b11_branch2c = mx.symbol.BatchNorm(name='bn4b11_branch2c', data=res4b11_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b11_branch2c = bn4b11_branch2c res4b11 = mx.symbol.broadcast_add(name='res4b11', *[res4b10_relu,scale4b11_branch2c] ) res4b11_relu = mx.symbol.Activation(name='res4b11_relu', data=res4b11 , act_type='relu') res4b12_branch2a = mx.symbol.Convolution(name='res4b12_branch2a', data=res4b11_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b12_branch2a = mx.symbol.BatchNorm(name='bn4b12_branch2a', data=res4b12_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b12_branch2a = bn4b12_branch2a res4b12_branch2a_relu = mx.symbol.Activation(name='res4b12_branch2a_relu', data=scale4b12_branch2a , act_type='relu') res4b12_branch2b = mx.symbol.Convolution(name='res4b12_branch2b', data=res4b12_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b12_branch2b = mx.symbol.BatchNorm(name='bn4b12_branch2b', data=res4b12_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b12_branch2b = bn4b12_branch2b res4b12_branch2b_relu = mx.symbol.Activation(name='res4b12_branch2b_relu', data=scale4b12_branch2b , act_type='relu') res4b12_branch2c = mx.symbol.Convolution(name='res4b12_branch2c', data=res4b12_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b12_branch2c = mx.symbol.BatchNorm(name='bn4b12_branch2c', data=res4b12_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b12_branch2c = bn4b12_branch2c res4b12 = mx.symbol.broadcast_add(name='res4b12', *[res4b11_relu,scale4b12_branch2c] ) res4b12_relu = mx.symbol.Activation(name='res4b12_relu', data=res4b12 , act_type='relu') res4b13_branch2a = mx.symbol.Convolution(name='res4b13_branch2a', data=res4b12_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b13_branch2a = mx.symbol.BatchNorm(name='bn4b13_branch2a', data=res4b13_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b13_branch2a = bn4b13_branch2a res4b13_branch2a_relu = mx.symbol.Activation(name='res4b13_branch2a_relu', data=scale4b13_branch2a , act_type='relu') res4b13_branch2b = mx.symbol.Convolution(name='res4b13_branch2b', data=res4b13_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b13_branch2b = mx.symbol.BatchNorm(name='bn4b13_branch2b', data=res4b13_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b13_branch2b = bn4b13_branch2b res4b13_branch2b_relu = mx.symbol.Activation(name='res4b13_branch2b_relu', data=scale4b13_branch2b , act_type='relu') res4b13_branch2c = mx.symbol.Convolution(name='res4b13_branch2c', data=res4b13_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b13_branch2c = mx.symbol.BatchNorm(name='bn4b13_branch2c', data=res4b13_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b13_branch2c = bn4b13_branch2c res4b13 = mx.symbol.broadcast_add(name='res4b13', *[res4b12_relu,scale4b13_branch2c] ) res4b13_relu = mx.symbol.Activation(name='res4b13_relu', data=res4b13 , act_type='relu') res4b14_branch2a = mx.symbol.Convolution(name='res4b14_branch2a', data=res4b13_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b14_branch2a = mx.symbol.BatchNorm(name='bn4b14_branch2a', data=res4b14_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b14_branch2a = bn4b14_branch2a res4b14_branch2a_relu = mx.symbol.Activation(name='res4b14_branch2a_relu', data=scale4b14_branch2a , act_type='relu') res4b14_branch2b = mx.symbol.Convolution(name='res4b14_branch2b', data=res4b14_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b14_branch2b = mx.symbol.BatchNorm(name='bn4b14_branch2b', data=res4b14_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b14_branch2b = bn4b14_branch2b res4b14_branch2b_relu = mx.symbol.Activation(name='res4b14_branch2b_relu', data=scale4b14_branch2b , act_type='relu') res4b14_branch2c = mx.symbol.Convolution(name='res4b14_branch2c', data=res4b14_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b14_branch2c = mx.symbol.BatchNorm(name='bn4b14_branch2c', data=res4b14_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b14_branch2c = bn4b14_branch2c res4b14 = mx.symbol.broadcast_add(name='res4b14', *[res4b13_relu,scale4b14_branch2c] ) res4b14_relu = mx.symbol.Activation(name='res4b14_relu', data=res4b14 , act_type='relu') res4b15_branch2a = mx.symbol.Convolution(name='res4b15_branch2a', data=res4b14_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b15_branch2a = mx.symbol.BatchNorm(name='bn4b15_branch2a', data=res4b15_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b15_branch2a = bn4b15_branch2a res4b15_branch2a_relu = mx.symbol.Activation(name='res4b15_branch2a_relu', data=scale4b15_branch2a , act_type='relu') res4b15_branch2b = mx.symbol.Convolution(name='res4b15_branch2b', data=res4b15_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b15_branch2b = mx.symbol.BatchNorm(name='bn4b15_branch2b', data=res4b15_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b15_branch2b = bn4b15_branch2b res4b15_branch2b_relu = mx.symbol.Activation(name='res4b15_branch2b_relu', data=scale4b15_branch2b , act_type='relu') res4b15_branch2c = mx.symbol.Convolution(name='res4b15_branch2c', data=res4b15_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b15_branch2c = mx.symbol.BatchNorm(name='bn4b15_branch2c', data=res4b15_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b15_branch2c = bn4b15_branch2c res4b15 = mx.symbol.broadcast_add(name='res4b15', *[res4b14_relu,scale4b15_branch2c] ) res4b15_relu = mx.symbol.Activation(name='res4b15_relu', data=res4b15 , act_type='relu') res4b16_branch2a = mx.symbol.Convolution(name='res4b16_branch2a', data=res4b15_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b16_branch2a = mx.symbol.BatchNorm(name='bn4b16_branch2a', data=res4b16_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b16_branch2a = bn4b16_branch2a res4b16_branch2a_relu = mx.symbol.Activation(name='res4b16_branch2a_relu', data=scale4b16_branch2a , act_type='relu') res4b16_branch2b = mx.symbol.Convolution(name='res4b16_branch2b', data=res4b16_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b16_branch2b = mx.symbol.BatchNorm(name='bn4b16_branch2b', data=res4b16_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b16_branch2b = bn4b16_branch2b res4b16_branch2b_relu = mx.symbol.Activation(name='res4b16_branch2b_relu', data=scale4b16_branch2b , act_type='relu') res4b16_branch2c = mx.symbol.Convolution(name='res4b16_branch2c', data=res4b16_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b16_branch2c = mx.symbol.BatchNorm(name='bn4b16_branch2c', data=res4b16_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b16_branch2c = bn4b16_branch2c res4b16 = mx.symbol.broadcast_add(name='res4b16', *[res4b15_relu,scale4b16_branch2c] ) res4b16_relu = mx.symbol.Activation(name='res4b16_relu', data=res4b16 , act_type='relu') res4b17_branch2a = mx.symbol.Convolution(name='res4b17_branch2a', data=res4b16_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b17_branch2a = mx.symbol.BatchNorm(name='bn4b17_branch2a', data=res4b17_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b17_branch2a = bn4b17_branch2a res4b17_branch2a_relu = mx.symbol.Activation(name='res4b17_branch2a_relu', data=scale4b17_branch2a , act_type='relu') res4b17_branch2b = mx.symbol.Convolution(name='res4b17_branch2b', data=res4b17_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b17_branch2b = mx.symbol.BatchNorm(name='bn4b17_branch2b', data=res4b17_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b17_branch2b = bn4b17_branch2b res4b17_branch2b_relu = mx.symbol.Activation(name='res4b17_branch2b_relu', data=scale4b17_branch2b , act_type='relu') res4b17_branch2c = mx.symbol.Convolution(name='res4b17_branch2c', data=res4b17_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b17_branch2c = mx.symbol.BatchNorm(name='bn4b17_branch2c', data=res4b17_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b17_branch2c = bn4b17_branch2c res4b17 = mx.symbol.broadcast_add(name='res4b17', *[res4b16_relu,scale4b17_branch2c] ) res4b17_relu = mx.symbol.Activation(name='res4b17_relu', data=res4b17 , act_type='relu') res4b18_branch2a = mx.symbol.Convolution(name='res4b18_branch2a', data=res4b17_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b18_branch2a = mx.symbol.BatchNorm(name='bn4b18_branch2a', data=res4b18_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b18_branch2a = bn4b18_branch2a res4b18_branch2a_relu = mx.symbol.Activation(name='res4b18_branch2a_relu', data=scale4b18_branch2a , act_type='relu') res4b18_branch2b = mx.symbol.Convolution(name='res4b18_branch2b', data=res4b18_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b18_branch2b = mx.symbol.BatchNorm(name='bn4b18_branch2b', data=res4b18_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b18_branch2b = bn4b18_branch2b res4b18_branch2b_relu = mx.symbol.Activation(name='res4b18_branch2b_relu', data=scale4b18_branch2b , act_type='relu') res4b18_branch2c = mx.symbol.Convolution(name='res4b18_branch2c', data=res4b18_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b18_branch2c = mx.symbol.BatchNorm(name='bn4b18_branch2c', data=res4b18_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b18_branch2c = bn4b18_branch2c res4b18 = mx.symbol.broadcast_add(name='res4b18', *[res4b17_relu,scale4b18_branch2c] ) res4b18_relu = mx.symbol.Activation(name='res4b18_relu', data=res4b18 , act_type='relu') res4b19_branch2a = mx.symbol.Convolution(name='res4b19_branch2a', data=res4b18_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b19_branch2a = mx.symbol.BatchNorm(name='bn4b19_branch2a', data=res4b19_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b19_branch2a = bn4b19_branch2a res4b19_branch2a_relu = mx.symbol.Activation(name='res4b19_branch2a_relu', data=scale4b19_branch2a , act_type='relu') res4b19_branch2b = mx.symbol.Convolution(name='res4b19_branch2b', data=res4b19_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b19_branch2b = mx.symbol.BatchNorm(name='bn4b19_branch2b', data=res4b19_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b19_branch2b = bn4b19_branch2b res4b19_branch2b_relu = mx.symbol.Activation(name='res4b19_branch2b_relu', data=scale4b19_branch2b , act_type='relu') res4b19_branch2c = mx.symbol.Convolution(name='res4b19_branch2c', data=res4b19_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b19_branch2c = mx.symbol.BatchNorm(name='bn4b19_branch2c', data=res4b19_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b19_branch2c = bn4b19_branch2c res4b19 = mx.symbol.broadcast_add(name='res4b19', *[res4b18_relu,scale4b19_branch2c] ) res4b19_relu = mx.symbol.Activation(name='res4b19_relu', data=res4b19 , act_type='relu') res4b20_branch2a = mx.symbol.Convolution(name='res4b20_branch2a', data=res4b19_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b20_branch2a = mx.symbol.BatchNorm(name='bn4b20_branch2a', data=res4b20_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b20_branch2a = bn4b20_branch2a res4b20_branch2a_relu = mx.symbol.Activation(name='res4b20_branch2a_relu', data=scale4b20_branch2a , act_type='relu') res4b20_branch2b = mx.symbol.Convolution(name='res4b20_branch2b', data=res4b20_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b20_branch2b = mx.symbol.BatchNorm(name='bn4b20_branch2b', data=res4b20_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b20_branch2b = bn4b20_branch2b res4b20_branch2b_relu = mx.symbol.Activation(name='res4b20_branch2b_relu', data=scale4b20_branch2b , act_type='relu') res4b20_branch2c = mx.symbol.Convolution(name='res4b20_branch2c', data=res4b20_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b20_branch2c = mx.symbol.BatchNorm(name='bn4b20_branch2c', data=res4b20_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b20_branch2c = bn4b20_branch2c res4b20 = mx.symbol.broadcast_add(name='res4b20', *[res4b19_relu,scale4b20_branch2c] ) res4b20_relu = mx.symbol.Activation(name='res4b20_relu', data=res4b20 , act_type='relu') res4b21_branch2a = mx.symbol.Convolution(name='res4b21_branch2a', data=res4b20_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b21_branch2a = mx.symbol.BatchNorm(name='bn4b21_branch2a', data=res4b21_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b21_branch2a = bn4b21_branch2a res4b21_branch2a_relu = mx.symbol.Activation(name='res4b21_branch2a_relu', data=scale4b21_branch2a , act_type='relu') res4b21_branch2b = mx.symbol.Convolution(name='res4b21_branch2b', data=res4b21_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b21_branch2b = mx.symbol.BatchNorm(name='bn4b21_branch2b', data=res4b21_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b21_branch2b = bn4b21_branch2b res4b21_branch2b_relu = mx.symbol.Activation(name='res4b21_branch2b_relu', data=scale4b21_branch2b , act_type='relu') res4b21_branch2c = mx.symbol.Convolution(name='res4b21_branch2c', data=res4b21_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b21_branch2c = mx.symbol.BatchNorm(name='bn4b21_branch2c', data=res4b21_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b21_branch2c = bn4b21_branch2c res4b21 = mx.symbol.broadcast_add(name='res4b21', *[res4b20_relu,scale4b21_branch2c] ) res4b21_relu = mx.symbol.Activation(name='res4b21_relu', data=res4b21 , act_type='relu') res4b22_branch2a = mx.symbol.Convolution(name='res4b22_branch2a', data=res4b21_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b22_branch2a = mx.symbol.BatchNorm(name='bn4b22_branch2a', data=res4b22_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b22_branch2a = bn4b22_branch2a res4b22_branch2a_relu = mx.symbol.Activation(name='res4b22_branch2a_relu', data=scale4b22_branch2a , act_type='relu') res4b22_branch2b = mx.symbol.Convolution(name='res4b22_branch2b', data=res4b22_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b22_branch2b = mx.symbol.BatchNorm(name='bn4b22_branch2b', data=res4b22_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b22_branch2b = bn4b22_branch2b res4b22_branch2b_relu = mx.symbol.Activation(name='res4b22_branch2b_relu', data=scale4b22_branch2b , act_type='relu') res4b22_branch2c = mx.symbol.Convolution(name='res4b22_branch2c', data=res4b22_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b22_branch2c = mx.symbol.BatchNorm(name='bn4b22_branch2c', data=res4b22_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b22_branch2c = bn4b22_branch2c res4b22 = mx.symbol.broadcast_add(name='res4b22', *[res4b21_relu,scale4b22_branch2c] ) res4b22_relu = mx.symbol.Activation(name='res4b22_relu', data=res4b22 , act_type='relu') res5a_branch1 = mx.symbol.Convolution(name='res5a_branch1', data=res4b22_relu , num_filter=2048, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn5a_branch1 = mx.symbol.BatchNorm(name='bn5a_branch1', data=res5a_branch1 , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale5a_branch1 = bn5a_branch1 res5a_branch2a = mx.symbol.Convolution(name='res5a_branch2a', data=res4b22_relu , num_filter=512, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn5a_branch2a = mx.symbol.BatchNorm(name='bn5a_branch2a', data=res5a_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale5a_branch2a = bn5a_branch2a res5a_branch2a_relu = mx.symbol.Activation(name='res5a_branch2a_relu', data=scale5a_branch2a , act_type='relu') res5a_branch2b = mx.symbol.Convolution(name='res5a_branch2b', data=res5a_branch2a_relu , num_filter=512, pad=(2,2), dilate=(2,2), kernel=(3,3), stride=(1,1), no_bias=True) bn5a_branch2b = mx.symbol.BatchNorm(name='bn5a_branch2b', data=res5a_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale5a_branch2b = bn5a_branch2b res5a_branch2b_relu = mx.symbol.Activation(name='res5a_branch2b_relu', data=scale5a_branch2b , act_type='relu') res5a_branch2c = mx.symbol.Convolution(name='res5a_branch2c', data=res5a_branch2b_relu , num_filter=2048, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn5a_branch2c = mx.symbol.BatchNorm(name='bn5a_branch2c', data=res5a_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale5a_branch2c = bn5a_branch2c res5a = mx.symbol.broadcast_add(name='res5a', *[scale5a_branch1,scale5a_branch2c] ) res5a_relu = mx.symbol.Activation(name='res5a_relu', data=res5a , act_type='relu') res5b_branch2a = mx.symbol.Convolution(name='res5b_branch2a', data=res5a_relu , num_filter=512, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn5b_branch2a = mx.symbol.BatchNorm(name='bn5b_branch2a', data=res5b_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale5b_branch2a = bn5b_branch2a res5b_branch2a_relu = mx.symbol.Activation(name='res5b_branch2a_relu', data=scale5b_branch2a , act_type='relu') res5b_branch2b = mx.symbol.Convolution(name='res5b_branch2b', data=res5b_branch2a_relu , num_filter=512, pad=(2,2), dilate=(2,2), kernel=(3,3), stride=(1,1), no_bias=True) bn5b_branch2b = mx.symbol.BatchNorm(name='bn5b_branch2b', data=res5b_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale5b_branch2b = bn5b_branch2b res5b_branch2b_relu = mx.symbol.Activation(name='res5b_branch2b_relu', data=scale5b_branch2b , act_type='relu') res5b_branch2c = mx.symbol.Convolution(name='res5b_branch2c', data=res5b_branch2b_relu , num_filter=2048, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn5b_branch2c = mx.symbol.BatchNorm(name='bn5b_branch2c', data=res5b_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale5b_branch2c = bn5b_branch2c res5b = mx.symbol.broadcast_add(name='res5b', *[res5a_relu,scale5b_branch2c] ) res5b_relu = mx.symbol.Activation(name='res5b_relu', data=res5b , act_type='relu') res5c_branch2a = mx.symbol.Convolution(name='res5c_branch2a', data=res5b_relu , num_filter=512, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn5c_branch2a = mx.symbol.BatchNorm(name='bn5c_branch2a', data=res5c_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale5c_branch2a = bn5c_branch2a res5c_branch2a_relu = mx.symbol.Activation(name='res5c_branch2a_relu', data=scale5c_branch2a , act_type='relu') res5c_branch2b = mx.symbol.Convolution(name='res5c_branch2b', data=res5c_branch2a_relu , num_filter=512, pad=(2,2), dilate=(2,2), kernel=(3,3), stride=(1,1), no_bias=True) bn5c_branch2b = mx.symbol.BatchNorm(name='bn5c_branch2b', data=res5c_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale5c_branch2b = bn5c_branch2b res5c_branch2b_relu = mx.symbol.Activation(name='res5c_branch2b_relu', data=scale5c_branch2b , act_type='relu') res5c_branch2c = mx.symbol.Convolution(name='res5c_branch2c', data=res5c_branch2b_relu , num_filter=2048, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn5c_branch2c = mx.symbol.BatchNorm(name='bn5c_branch2c', data=res5c_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale5c_branch2c = bn5c_branch2c res5c = mx.symbol.broadcast_add(name='res5c', *[res5b_relu,scale5c_branch2c] ) res5c_relu = mx.symbol.Activation(name='res5c_relu', data=res5c , act_type='relu') feat_conv_3x3 = mx.sym.Convolution( data=res5c_relu, kernel=(3, 3), pad=(6, 6), dilate=(6, 6), num_filter=1024, name="feat_conv_3x3") feat_conv_3x3_relu = mx.sym.Activation(data=feat_conv_3x3, act_type="relu", name="feat_conv_3x3_relu") return feat_conv_3x3_relu def get_flownet(self, img_cur, img_ref): data = mx.symbol.Concat(img_cur / 255.0, img_ref / 255.0, dim=1) resize_data = mx.symbol.Pooling(name='resize_data', data=data , pooling_convention='full', pad=(0,0), kernel=(2,2), stride=(2,2), pool_type='avg') flow_conv1 = mx.symbol.Convolution(name='flow_conv1', data=resize_data , num_filter=64, pad=(3,3), kernel=(7,7), stride=(2,2), no_bias=False) ReLU1 = mx.symbol.LeakyReLU(name='ReLU1', data=flow_conv1 , act_type='leaky', slope=0.1) conv2 = mx.symbol.Convolution(name='conv2', data=ReLU1 , num_filter=128, pad=(2,2), kernel=(5,5), stride=(2,2), no_bias=False) ReLU2 = mx.symbol.LeakyReLU(name='ReLU2', data=conv2 , act_type='leaky', slope=0.1) conv3 = mx.symbol.Convolution(name='conv3', data=ReLU2 , num_filter=256, pad=(2,2), kernel=(5,5), stride=(2,2), no_bias=False) ReLU3 = mx.symbol.LeakyReLU(name='ReLU3', data=conv3 , act_type='leaky', slope=0.1) conv3_1 = mx.symbol.Convolution(name='conv3_1', data=ReLU3 , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=False) ReLU4 = mx.symbol.LeakyReLU(name='ReLU4', data=conv3_1 , act_type='leaky', slope=0.1) conv4 = mx.symbol.Convolution(name='conv4', data=ReLU4 , num_filter=512, pad=(1,1), kernel=(3,3), stride=(2,2), no_bias=False) ReLU5 = mx.symbol.LeakyReLU(name='ReLU5', data=conv4 , act_type='leaky', slope=0.1) conv4_1 = mx.symbol.Convolution(name='conv4_1', data=ReLU5 , num_filter=512, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=False) ReLU6 = mx.symbol.LeakyReLU(name='ReLU6', data=conv4_1 , act_type='leaky', slope=0.1) conv5 = mx.symbol.Convolution(name='conv5', data=ReLU6 , num_filter=512, pad=(1,1), kernel=(3,3), stride=(2,2), no_bias=False) ReLU7 = mx.symbol.LeakyReLU(name='ReLU7', data=conv5 , act_type='leaky', slope=0.1) conv5_1 = mx.symbol.Convolution(name='conv5_1', data=ReLU7 , num_filter=512, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=False) ReLU8 = mx.symbol.LeakyReLU(name='ReLU8', data=conv5_1 , act_type='leaky', slope=0.1) conv6 = mx.symbol.Convolution(name='conv6', data=ReLU8 , num_filter=1024, pad=(1,1), kernel=(3,3), stride=(2,2), no_bias=False) ReLU9 = mx.symbol.LeakyReLU(name='ReLU9', data=conv6 , act_type='leaky', slope=0.1) conv6_1 = mx.symbol.Convolution(name='conv6_1', data=ReLU9 , num_filter=1024, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=False) ReLU10 = mx.symbol.LeakyReLU(name='ReLU10', data=conv6_1 , act_type='leaky', slope=0.1) Convolution1 = mx.symbol.Convolution(name='Convolution1', data=ReLU10 , num_filter=2, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=False) deconv5 = mx.symbol.Deconvolution(name='deconv5', data=ReLU10 , num_filter=512, pad=(0,0), kernel=(4,4), stride=(2,2), no_bias=False) crop_deconv5 = mx.symbol.Crop(name='crop_deconv5', *[deconv5,ReLU8] , offset=(1,1)) ReLU11 = mx.symbol.LeakyReLU(name='ReLU11', data=crop_deconv5 , act_type='leaky', slope=0.1) upsample_flow6to5 = mx.symbol.Deconvolution(name='upsample_flow6to5', data=Convolution1 , num_filter=2, pad=(0,0), kernel=(4,4), stride=(2,2), no_bias=False) crop_upsampled_flow6_to_5 = mx.symbol.Crop(name='crop_upsampled_flow6_to_5', *[upsample_flow6to5,ReLU8] , offset=(1,1)) Concat2 = mx.symbol.Concat(name='Concat2', *[ReLU8,ReLU11,crop_upsampled_flow6_to_5] ) Convolution2 = mx.symbol.Convolution(name='Convolution2', data=Concat2 , num_filter=2, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=False) deconv4 = mx.symbol.Deconvolution(name='deconv4', data=Concat2 , num_filter=256, pad=(0,0), kernel=(4,4), stride=(2,2), no_bias=False) crop_deconv4 = mx.symbol.Crop(name='crop_deconv4', *[deconv4,ReLU6] , offset=(1,1)) ReLU12 = mx.symbol.LeakyReLU(name='ReLU12', data=crop_deconv4 , act_type='leaky', slope=0.1) upsample_flow5to4 = mx.symbol.Deconvolution(name='upsample_flow5to4', data=Convolution2 , num_filter=2, pad=(0,0), kernel=(4,4), stride=(2,2), no_bias=False) crop_upsampled_flow5_to_4 = mx.symbol.Crop(name='crop_upsampled_flow5_to_4', *[upsample_flow5to4,ReLU6] , offset=(1,1)) Concat3 = mx.symbol.Concat(name='Concat3', *[ReLU6,ReLU12,crop_upsampled_flow5_to_4] ) Convolution3 = mx.symbol.Convolution(name='Convolution3', data=Concat3 , num_filter=2, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=False) deconv3 = mx.symbol.Deconvolution(name='deconv3', data=Concat3 , num_filter=128, pad=(0,0), kernel=(4,4), stride=(2,2), no_bias=False) crop_deconv3 = mx.symbol.Crop(name='crop_deconv3', *[deconv3,ReLU4] , offset=(1,1)) ReLU13 = mx.symbol.LeakyReLU(name='ReLU13', data=crop_deconv3 , act_type='leaky', slope=0.1) upsample_flow4to3 = mx.symbol.Deconvolution(name='upsample_flow4to3', data=Convolution3 , num_filter=2, pad=(0,0), kernel=(4,4), stride=(2,2), no_bias=False) crop_upsampled_flow4_to_3 = mx.symbol.Crop(name='crop_upsampled_flow4_to_3', *[upsample_flow4to3,ReLU4] , offset=(1,1)) Concat4 = mx.symbol.Concat(name='Concat4', *[ReLU4,ReLU13,crop_upsampled_flow4_to_3] ) Convolution4 = mx.symbol.Convolution(name='Convolution4', data=Concat4 , num_filter=2, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=False) deconv2 = mx.symbol.Deconvolution(name='deconv2', data=Concat4 , num_filter=64, pad=(0,0), kernel=(4,4), stride=(2,2), no_bias=False) crop_deconv2 = mx.symbol.Crop(name='crop_deconv2', *[deconv2,ReLU2] , offset=(1,1)) ReLU14 = mx.symbol.LeakyReLU(name='ReLU14', data=crop_deconv2 , act_type='leaky', slope=0.1) upsample_flow3to2 = mx.symbol.Deconvolution(name='upsample_flow3to2', data=Convolution4 , num_filter=2, pad=(0,0), kernel=(4,4), stride=(2,2), no_bias=False) crop_upsampled_flow3_to_2 = mx.symbol.Crop(name='crop_upsampled_flow3_to_2', *[upsample_flow3to2,ReLU2] , offset=(1,1)) Concat5 = mx.symbol.Concat(name='Concat5', *[ReLU2,ReLU14,crop_upsampled_flow3_to_2] ) Concat5 = mx.symbol.Pooling(name='resize_concat5', data=Concat5 , pooling_convention='full', pad=(0,0), kernel=(2,2), stride=(2,2), pool_type='avg') Convolution5 = mx.symbol.Convolution(name='Convolution5', data=Concat5 , num_filter=2, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=False) Convolution5_scale_bias = mx.sym.Variable(name='Convolution5_scale_bias', lr_mult=0.0) Convolution5_scale = mx.symbol.Convolution(name='Convolution5_scale', data=Concat5 , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), bias=Convolution5_scale_bias, no_bias=False) return Convolution5 * 2.5, Convolution5_scale def get_train_symbol(self, cfg): # config alias for convenient num_classes = cfg.dataset.NUM_CLASSES num_reg_classes = (2 if cfg.CLASS_AGNOSTIC else num_classes) num_anchors = cfg.network.NUM_ANCHORS data = mx.sym.Variable(name="data") data_ref = mx.sym.Variable(name="data_ref") eq_flag = mx.sym.Variable(name="eq_flag") im_info = mx.sym.Variable(name="im_info") gt_boxes = mx.sym.Variable(name="gt_boxes") rpn_label = mx.sym.Variable(name='label') rpn_bbox_target = mx.sym.Variable(name='bbox_target') rpn_bbox_weight = mx.sym.Variable(name='bbox_weight') # shared convolutional layers conv_feat = self.get_resnet_v1(data_ref) flow, scale_map = self.get_flownet(data, data_ref) flow_grid = mx.sym.GridGenerator(data=flow, transform_type='warp', name='flow_grid') warp_conv_feat = mx.sym.BilinearSampler(data=conv_feat, grid=flow_grid, name='warping_feat') warp_conv_feat = warp_conv_feat * scale_map select_conv_feat = mx.sym.take(mx.sym.Concat(*[warp_conv_feat, conv_feat], dim=0), eq_flag) conv_feats = mx.sym.SliceChannel(select_conv_feat, axis=1, num_outputs=2) # RPN layers rpn_feat = conv_feats[0] rpn_cls_score = mx.sym.Convolution( data=rpn_feat, kernel=(1, 1), pad=(0, 0), num_filter=2 * num_anchors, name="rpn_cls_score") rpn_bbox_pred = mx.sym.Convolution( data=rpn_feat, kernel=(1, 1), pad=(0, 0), num_filter=4 * num_anchors, name="rpn_bbox_pred") # prepare rpn data rpn_cls_score_reshape = mx.sym.Reshape( data=rpn_cls_score, shape=(0, 2, -1, 0), name="rpn_cls_score_reshape") # classification rpn_cls_prob = mx.sym.SoftmaxOutput(data=rpn_cls_score_reshape, label=rpn_label, multi_output=True, normalization='valid', use_ignore=True, ignore_label=-1, name="rpn_cls_prob") # bounding box regression if cfg.network.NORMALIZE_RPN: rpn_bbox_loss_ = rpn_bbox_weight * mx.sym.smooth_l1(name='rpn_bbox_loss_', scalar=1.0, data=(rpn_bbox_pred - rpn_bbox_target)) rpn_bbox_pred = mx.sym.Custom( bbox_pred=rpn_bbox_pred, op_type='rpn_inv_normalize', num_anchors=num_anchors, bbox_mean=cfg.network.ANCHOR_MEANS, bbox_std=cfg.network.ANCHOR_STDS) else: rpn_bbox_loss_ = rpn_bbox_weight * mx.sym.smooth_l1(name='rpn_bbox_loss_', scalar=3.0, data=(rpn_bbox_pred - rpn_bbox_target)) rpn_bbox_loss = mx.sym.MakeLoss(name='rpn_bbox_loss', data=rpn_bbox_loss_, grad_scale=1.0 / cfg.TRAIN.RPN_BATCH_SIZE) # ROI proposal rpn_cls_act = mx.sym.SoftmaxActivation( data=rpn_cls_score_reshape, mode="channel", name="rpn_cls_act") rpn_cls_act_reshape = mx.sym.Reshape( data=rpn_cls_act, shape=(0, 2 * num_anchors, -1, 0), name='rpn_cls_act_reshape') if cfg.TRAIN.CXX_PROPOSAL: rois = mx.contrib.sym.Proposal( cls_prob=rpn_cls_act_reshape, bbox_pred=rpn_bbox_pred, im_info=im_info, name='rois', feature_stride=cfg.network.RPN_FEAT_STRIDE, scales=tuple(cfg.network.ANCHOR_SCALES), ratios=tuple(cfg.network.ANCHOR_RATIOS), rpn_pre_nms_top_n=cfg.TRAIN.RPN_PRE_NMS_TOP_N, rpn_post_nms_top_n=cfg.TRAIN.RPN_POST_NMS_TOP_N, threshold=cfg.TRAIN.RPN_NMS_THRESH, rpn_min_size=cfg.TRAIN.RPN_MIN_SIZE) else: rois = mx.sym.Custom( cls_prob=rpn_cls_act_reshape, bbox_pred=rpn_bbox_pred, im_info=im_info, name='rois', op_type='proposal', feat_stride=cfg.network.RPN_FEAT_STRIDE, scales=tuple(cfg.network.ANCHOR_SCALES), ratios=tuple(cfg.network.ANCHOR_RATIOS), rpn_pre_nms_top_n=cfg.TRAIN.RPN_PRE_NMS_TOP_N, rpn_post_nms_top_n=cfg.TRAIN.RPN_POST_NMS_TOP_N, threshold=cfg.TRAIN.RPN_NMS_THRESH, rpn_min_size=cfg.TRAIN.RPN_MIN_SIZE) # ROI proposal target gt_boxes_reshape = mx.sym.Reshape(data=gt_boxes, shape=(-1, 5), name='gt_boxes_reshape') rois, label, bbox_target, bbox_weight = mx.sym.Custom(rois=rois, gt_boxes=gt_boxes_reshape, op_type='proposal_target', num_classes=num_reg_classes, batch_images=cfg.TRAIN.BATCH_IMAGES, batch_rois=cfg.TRAIN.BATCH_ROIS, cfg=cPickle.dumps(cfg), fg_fraction=cfg.TRAIN.FG_FRACTION) # res5 rfcn_feat = conv_feats[1] rfcn_cls = mx.sym.Convolution(data=rfcn_feat, kernel=(1, 1), num_filter=7*7*num_classes, name="rfcn_cls") rfcn_bbox = mx.sym.Convolution(data=rfcn_feat, kernel=(1, 1), num_filter=7*7*4*num_reg_classes, name="rfcn_bbox") psroipooled_cls_rois = mx.contrib.sym.PSROIPooling(name='psroipooled_cls_rois', data=rfcn_cls, rois=rois, group_size=7,pooled_size=7, output_dim=num_classes, spatial_scale=0.0625) psroipooled_loc_rois = mx.contrib.sym.PSROIPooling(name='psroipooled_loc_rois', data=rfcn_bbox, rois=rois, group_size=7,pooled_size=7, output_dim=8, spatial_scale=0.0625) cls_score = mx.sym.Pooling(name='ave_cls_scors_rois', data=psroipooled_cls_rois, pool_type='avg', global_pool=True, kernel=(7, 7)) bbox_pred = mx.sym.Pooling(name='ave_bbox_pred_rois', data=psroipooled_loc_rois, pool_type='avg', global_pool=True, kernel=(7, 7)) cls_score = mx.sym.Reshape(name='cls_score_reshape', data=cls_score, shape=(-1, num_classes)) bbox_pred = mx.sym.Reshape(name='bbox_pred_reshape', data=bbox_pred, shape=(-1, 4 * num_reg_classes)) # classification if cfg.TRAIN.ENABLE_OHEM: print 'use ohem!' labels_ohem, bbox_weights_ohem = mx.sym.Custom(op_type='BoxAnnotatorOHEM', num_classes=num_classes, num_reg_classes=num_reg_classes, roi_per_img=cfg.TRAIN.BATCH_ROIS_OHEM, cls_score=cls_score, bbox_pred=bbox_pred, labels=label, bbox_targets=bbox_target, bbox_weights=bbox_weight) cls_prob = mx.sym.SoftmaxOutput(name='cls_prob', data=cls_score, label=labels_ohem, normalization='valid', use_ignore=True, ignore_label=-1) bbox_loss_ = bbox_weights_ohem * mx.sym.smooth_l1(name='bbox_loss_', scalar=1.0, data=(bbox_pred - bbox_target)) bbox_loss = mx.sym.MakeLoss(name='bbox_loss', data=bbox_loss_, grad_scale=1.0 / cfg.TRAIN.BATCH_ROIS_OHEM) rcnn_label = labels_ohem else: cls_prob = mx.sym.SoftmaxOutput(name='cls_prob', data=cls_score, label=label, normalization='valid') bbox_loss_ = bbox_weight * mx.sym.smooth_l1(name='bbox_loss_', scalar=1.0, data=(bbox_pred - bbox_target)) bbox_loss = mx.sym.MakeLoss(name='bbox_loss', data=bbox_loss_, grad_scale=1.0 / cfg.TRAIN.BATCH_ROIS) rcnn_label = label # reshape output rcnn_label = mx.sym.Reshape(data=rcnn_label, shape=(cfg.TRAIN.BATCH_IMAGES, -1), name='label_reshape') cls_prob = mx.sym.Reshape(data=cls_prob, shape=(cfg.TRAIN.BATCH_IMAGES, -1, num_classes), name='cls_prob_reshape') bbox_loss = mx.sym.Reshape(data=bbox_loss, shape=(cfg.TRAIN.BATCH_IMAGES, -1, 4 * num_reg_classes), name='bbox_loss_reshape') group = mx.sym.Group([rpn_cls_prob, rpn_bbox_loss, cls_prob, bbox_loss, mx.sym.BlockGrad(rcnn_label)]) self.sym = group return group def get_key_test_symbol(self, cfg): # config alias for convenient num_classes = cfg.dataset.NUM_CLASSES num_reg_classes = (2 if cfg.CLASS_AGNOSTIC else num_classes) num_anchors = cfg.network.NUM_ANCHORS data = mx.sym.Variable(name="data") im_info = mx.sym.Variable(name="im_info") data_key = mx.sym.Variable(name="data_key") feat_key = mx.sym.Variable(name="feat_key") # shared convolutional layers conv_feat = self.get_resnet_v1(data) conv_feats = mx.sym.SliceChannel(conv_feat, axis=1, num_outputs=2) # RPN rpn_feat = conv_feats[0] rpn_cls_score = mx.sym.Convolution( data=rpn_feat, kernel=(1, 1), pad=(0, 0), num_filter=2 * num_anchors, name="rpn_cls_score") rpn_bbox_pred = mx.sym.Convolution( data=rpn_feat, kernel=(1, 1), pad=(0, 0), num_filter=4 * num_anchors, name="rpn_bbox_pred") if cfg.network.NORMALIZE_RPN: rpn_bbox_pred = mx.sym.Custom( bbox_pred=rpn_bbox_pred, op_type='rpn_inv_normalize', num_anchors=num_anchors, bbox_mean=cfg.network.ANCHOR_MEANS, bbox_std=cfg.network.ANCHOR_STDS) # ROI Proposal rpn_cls_score_reshape = mx.sym.Reshape( data=rpn_cls_score, shape=(0, 2, -1, 0), name="rpn_cls_score_reshape") rpn_cls_prob = mx.sym.SoftmaxActivation( data=rpn_cls_score_reshape, mode="channel", name="rpn_cls_prob") rpn_cls_prob_reshape = mx.sym.Reshape( data=rpn_cls_prob, shape=(0, 2 * num_anchors, -1, 0), name='rpn_cls_prob_reshape') if cfg.TEST.CXX_PROPOSAL: rois = mx.contrib.sym.Proposal( cls_prob=rpn_cls_prob_reshape, bbox_pred=rpn_bbox_pred, im_info=im_info, name='rois', feature_stride=cfg.network.RPN_FEAT_STRIDE, scales=tuple(cfg.network.ANCHOR_SCALES), ratios=tuple(cfg.network.ANCHOR_RATIOS), rpn_pre_nms_top_n=cfg.TEST.RPN_PRE_NMS_TOP_N, rpn_post_nms_top_n=cfg.TEST.RPN_POST_NMS_TOP_N, threshold=cfg.TEST.RPN_NMS_THRESH, rpn_min_size=cfg.TEST.RPN_MIN_SIZE) else: rois = mx.sym.Custom( cls_prob=rpn_cls_prob_reshape, bbox_pred=rpn_bbox_pred, im_info=im_info, name='rois', op_type='proposal', feat_stride=cfg.network.RPN_FEAT_STRIDE, scales=tuple(cfg.network.ANCHOR_SCALES), ratios=tuple(cfg.network.ANCHOR_RATIOS), rpn_pre_nms_top_n=cfg.TEST.RPN_PRE_NMS_TOP_N, rpn_post_nms_top_n=cfg.TEST.RPN_POST_NMS_TOP_N, threshold=cfg.TEST.RPN_NMS_THRESH, rpn_min_size=cfg.TEST.RPN_MIN_SIZE) # res5 rfcn_feat = conv_feats[1] rfcn_cls = mx.sym.Convolution(data=rfcn_feat, kernel=(1, 1), num_filter=7*7*num_classes, name="rfcn_cls") rfcn_bbox = mx.sym.Convolution(data=rfcn_feat, kernel=(1, 1), num_filter=7*7*4*num_reg_classes, name="rfcn_bbox") psroipooled_cls_rois = mx.contrib.sym.PSROIPooling(name='psroipooled_cls_rois', data=rfcn_cls, rois=rois, group_size=7, pooled_size=7, output_dim=num_classes, spatial_scale=0.0625) psroipooled_loc_rois = mx.contrib.sym.PSROIPooling(name='psroipooled_loc_rois', data=rfcn_bbox, rois=rois, group_size=7, pooled_size=7, output_dim=8, spatial_scale=0.0625) cls_score = mx.sym.Pooling(name='ave_cls_scors_rois', data=psroipooled_cls_rois, pool_type='avg', global_pool=True, kernel=(7, 7)) bbox_pred = mx.sym.Pooling(name='ave_bbox_pred_rois', data=psroipooled_loc_rois, pool_type='avg', global_pool=True, kernel=(7, 7)) # classification cls_score = mx.sym.Reshape(name='cls_score_reshape', data=cls_score, shape=(-1, num_classes)) cls_prob = mx.sym.SoftmaxActivation(name='cls_prob', data=cls_score) # bounding box regression bbox_pred = mx.sym.Reshape(name='bbox_pred_reshape', data=bbox_pred, shape=(-1, 4 * num_reg_classes)) # reshape output cls_prob = mx.sym.Reshape(data=cls_prob, shape=(cfg.TEST.BATCH_IMAGES, -1, num_classes), name='cls_prob_reshape') bbox_pred = mx.sym.Reshape(data=bbox_pred, shape=(cfg.TEST.BATCH_IMAGES, -1, 4 * num_reg_classes), name='bbox_pred_reshape') # group output group = mx.sym.Group([data_key, feat_key, conv_feat, rois, cls_prob, bbox_pred]) self.sym = group return group def get_cur_test_symbol(self, cfg): # config alias for convenient num_classes = cfg.dataset.NUM_CLASSES num_reg_classes = (2 if cfg.CLASS_AGNOSTIC else num_classes) num_anchors = cfg.network.NUM_ANCHORS data_cur = mx.sym.Variable(name="data") im_info = mx.sym.Variable(name="im_info") data_key = mx.sym.Variable(name="data_key") conv_feat = mx.sym.Variable(name="feat_key") # shared convolutional layers flow, scale_map = self.get_flownet(data_cur, data_key) flow_grid = mx.sym.GridGenerator(data=flow, transform_type='warp', name='flow_grid') conv_feat = mx.sym.BilinearSampler(data=conv_feat, grid=flow_grid, name='warping_feat') conv_feat = conv_feat * scale_map conv_feats = mx.sym.SliceChannel(conv_feat, axis=1, num_outputs=2) # RPN rpn_feat = conv_feats[0] rpn_cls_score = mx.sym.Convolution( data=rpn_feat, kernel=(1, 1), pad=(0, 0), num_filter=2 * num_anchors, name="rpn_cls_score") rpn_bbox_pred = mx.sym.Convolution( data=rpn_feat, kernel=(1, 1), pad=(0, 0), num_filter=4 * num_anchors, name="rpn_bbox_pred") if cfg.network.NORMALIZE_RPN: rpn_bbox_pred = mx.sym.Custom( bbox_pred=rpn_bbox_pred, op_type='rpn_inv_normalize', num_anchors=num_anchors, bbox_mean=cfg.network.ANCHOR_MEANS, bbox_std=cfg.network.ANCHOR_STDS) # ROI Proposal rpn_cls_score_reshape = mx.sym.Reshape( data=rpn_cls_score, shape=(0, 2, -1, 0), name="rpn_cls_score_reshape") rpn_cls_prob = mx.sym.SoftmaxActivation( data=rpn_cls_score_reshape, mode="channel", name="rpn_cls_prob") rpn_cls_prob_reshape = mx.sym.Reshape( data=rpn_cls_prob, shape=(0, 2 * num_anchors, -1, 0), name='rpn_cls_prob_reshape') if cfg.TEST.CXX_PROPOSAL: rois = mx.contrib.sym.Proposal( cls_prob=rpn_cls_prob_reshape, bbox_pred=rpn_bbox_pred, im_info=im_info, name='rois', feature_stride=cfg.network.RPN_FEAT_STRIDE, scales=tuple(cfg.network.ANCHOR_SCALES), ratios=tuple(cfg.network.ANCHOR_RATIOS), rpn_pre_nms_top_n=cfg.TEST.RPN_PRE_NMS_TOP_N, rpn_post_nms_top_n=cfg.TEST.RPN_POST_NMS_TOP_N, threshold=cfg.TEST.RPN_NMS_THRESH, rpn_min_size=cfg.TEST.RPN_MIN_SIZE) else: rois = mx.sym.Custom( cls_prob=rpn_cls_prob_reshape, bbox_pred=rpn_bbox_pred, im_info=im_info, name='rois', op_type='proposal', feat_stride=cfg.network.RPN_FEAT_STRIDE, scales=tuple(cfg.network.ANCHOR_SCALES), ratios=tuple(cfg.network.ANCHOR_RATIOS), rpn_pre_nms_top_n=cfg.TEST.RPN_PRE_NMS_TOP_N, rpn_post_nms_top_n=cfg.TEST.RPN_POST_NMS_TOP_N, threshold=cfg.TEST.RPN_NMS_THRESH, rpn_min_size=cfg.TEST.RPN_MIN_SIZE) # res5 rfcn_feat = conv_feats[1] rfcn_cls = mx.sym.Convolution(data=rfcn_feat, kernel=(1, 1), num_filter=7*7*num_classes, name="rfcn_cls") rfcn_bbox = mx.sym.Convolution(data=rfcn_feat, kernel=(1, 1), num_filter=7*7*4*num_reg_classes, name="rfcn_bbox") psroipooled_cls_rois = mx.contrib.sym.PSROIPooling(name='psroipooled_cls_rois', data=rfcn_cls, rois=rois, group_size=7, pooled_size=7, output_dim=num_classes, spatial_scale=0.0625) psroipooled_loc_rois = mx.contrib.sym.PSROIPooling(name='psroipooled_loc_rois', data=rfcn_bbox, rois=rois, group_size=7, pooled_size=7, output_dim=8, spatial_scale=0.0625) cls_score = mx.sym.Pooling(name='ave_cls_scors_rois', data=psroipooled_cls_rois, pool_type='avg', global_pool=True, kernel=(7, 7)) bbox_pred = mx.sym.Pooling(name='ave_bbox_pred_rois', data=psroipooled_loc_rois, pool_type='avg', global_pool=True, kernel=(7, 7)) # classification cls_score = mx.sym.Reshape(name='cls_score_reshape', data=cls_score, shape=(-1, num_classes)) cls_prob = mx.sym.SoftmaxActivation(name='cls_prob', data=cls_score) # bounding box regression bbox_pred = mx.sym.Reshape(name='bbox_pred_reshape', data=bbox_pred, shape=(-1, 4 * num_reg_classes)) # reshape output cls_prob = mx.sym.Reshape(data=cls_prob, shape=(cfg.TEST.BATCH_IMAGES, -1, num_classes), name='cls_prob_reshape') bbox_pred = mx.sym.Reshape(data=bbox_pred, shape=(cfg.TEST.BATCH_IMAGES, -1, 4 * num_reg_classes), name='bbox_pred_reshape') # group output group = mx.sym.Group([rois, cls_prob, bbox_pred]) self.sym = group return group def get_batch_test_symbol(self, cfg): # config alias for convenient num_classes = cfg.dataset.NUM_CLASSES num_reg_classes = (2 if cfg.CLASS_AGNOSTIC else num_classes) num_anchors = cfg.network.NUM_ANCHORS data_key = mx.sym.Variable(name="data_key") data_other = mx.sym.Variable(name="data_other") im_info = mx.sym.Variable(name="im_info") # shared convolutional layers conv_feat_key = self.get_resnet_v1(data_key) data_key_tiled = mx.sym.Custom(data_content=data_key, data_shape=data_other, op_type='tile_as') conv_feat_key_tiled = mx.sym.Custom(data_content=conv_feat_key, data_shape=data_other, op_type='tile_as') flow, scale_map = self.get_flownet(data_other, data_key_tiled) flow_grid = mx.sym.GridGenerator(data=flow, transform_type='warp', name='flow_grid') conv_feat_other = mx.sym.BilinearSampler(data=conv_feat_key_tiled, grid=flow_grid, name='warping_feat') conv_feat_other = conv_feat_other * scale_map conv_feat = mx.symbol.Concat(conv_feat_key, conv_feat_other, dim=0) conv_feats = mx.sym.SliceChannel(conv_feat, axis=1, num_outputs=2) # RPN rpn_feat = conv_feats[0] rpn_cls_score = mx.sym.Convolution( data=rpn_feat, kernel=(1, 1), pad=(0, 0), num_filter=2 * num_anchors, name="rpn_cls_score") rpn_bbox_pred = mx.sym.Convolution( data=rpn_feat, kernel=(1, 1), pad=(0, 0), num_filter=4 * num_anchors, name="rpn_bbox_pred") if cfg.network.NORMALIZE_RPN: rpn_bbox_pred = mx.sym.Custom( bbox_pred=rpn_bbox_pred, op_type='rpn_inv_normalize', num_anchors=num_anchors, bbox_mean=cfg.network.ANCHOR_MEANS, bbox_std=cfg.network.ANCHOR_STDS) # ROI Proposal rpn_cls_score_reshape = mx.sym.Reshape( data=rpn_cls_score, shape=(0, 2, -1, 0), name="rpn_cls_score_reshape") rpn_cls_prob = mx.sym.SoftmaxActivation( data=rpn_cls_score_reshape, mode="channel", name="rpn_cls_prob") rpn_cls_prob_reshape = mx.sym.Reshape( data=rpn_cls_prob, shape=(0, 2 * num_anchors, -1, 0), name='rpn_cls_prob_reshape') if cfg.TEST.CXX_PROPOSAL: rois = mx.contrib.sym.MultiProposal( cls_prob=rpn_cls_prob_reshape, bbox_pred=rpn_bbox_pred, im_info=im_info, name='rois', feature_stride=cfg.network.RPN_FEAT_STRIDE, scales=tuple(cfg.network.ANCHOR_SCALES), ratios=tuple(cfg.network.ANCHOR_RATIOS), rpn_pre_nms_top_n=cfg.TEST.RPN_PRE_NMS_TOP_N, rpn_post_nms_top_n=cfg.TEST.RPN_POST_NMS_TOP_N, threshold=cfg.TEST.RPN_NMS_THRESH, rpn_min_size=cfg.TEST.RPN_MIN_SIZE) else: NotImplemented # res5 rfcn_feat = conv_feats[1] rfcn_cls = mx.sym.Convolution(data=rfcn_feat, kernel=(1, 1), num_filter=7*7*num_classes, name="rfcn_cls") rfcn_bbox = mx.sym.Convolution(data=rfcn_feat, kernel=(1, 1), num_filter=7*7*4*num_reg_classes, name="rfcn_bbox") psroipooled_cls_rois = mx.contrib.sym.PSROIPooling(name='psroipooled_cls_rois', data=rfcn_cls, rois=rois, group_size=7, pooled_size=7, output_dim=num_classes, spatial_scale=0.0625) psroipooled_loc_rois = mx.contrib.sym.PSROIPooling(name='psroipooled_loc_rois', data=rfcn_bbox, rois=rois, group_size=7, pooled_size=7, output_dim=8, spatial_scale=0.0625) cls_score = mx.sym.Pooling(name='ave_cls_scors_rois', data=psroipooled_cls_rois, pool_type='avg', global_pool=True, kernel=(7, 7)) bbox_pred = mx.sym.Pooling(name='ave_bbox_pred_rois', data=psroipooled_loc_rois, pool_type='avg', global_pool=True, kernel=(7, 7)) # classification cls_score = mx.sym.Reshape(name='cls_score_reshape', data=cls_score, shape=(-1, num_classes)) cls_prob = mx.sym.SoftmaxActivation(name='cls_prob', data=cls_score) # bounding box regression bbox_pred = mx.sym.Reshape(name='bbox_pred_reshape', data=bbox_pred, shape=(-1, 4 * num_reg_classes)) # reshape output cls_prob = mx.sym.Reshape(data=cls_prob, shape=(cfg.TEST.BATCH_IMAGES, -1, num_classes), name='cls_prob_reshape') bbox_pred = mx.sym.Reshape(data=bbox_pred, shape=(cfg.TEST.BATCH_IMAGES, -1, 4 * num_reg_classes), name='bbox_pred_reshape') # group output group = mx.sym.Group([rois, cls_prob, bbox_pred]) self.sym = group return group def init_weight(self, cfg, arg_params, aux_params): arg_params['Convolution5_scale_weight'] = mx.nd.zeros(shape=self.arg_shape_dict['Convolution5_scale_weight']) arg_params['Convolution5_scale_bias'] = mx.nd.ones(shape=self.arg_shape_dict['Convolution5_scale_bias']) arg_params['feat_conv_3x3_weight'] = mx.random.normal(0, 0.01, shape=self.arg_shape_dict['feat_conv_3x3_weight']) arg_params['feat_conv_3x3_bias'] = mx.nd.zeros(shape=self.arg_shape_dict['feat_conv_3x3_bias']) arg_params['rpn_cls_score_weight'] = mx.random.normal(0, 0.01, shape=self.arg_shape_dict['rpn_cls_score_weight']) arg_params['rpn_cls_score_bias'] = mx.nd.zeros(shape=self.arg_shape_dict['rpn_cls_score_bias']) arg_params['rpn_bbox_pred_weight'] = mx.random.normal(0, 0.01, shape=self.arg_shape_dict['rpn_bbox_pred_weight']) arg_params['rpn_bbox_pred_bias'] = mx.nd.zeros(shape=self.arg_shape_dict['rpn_bbox_pred_bias']) arg_params['rfcn_cls_weight'] = mx.random.normal(0, 0.01, shape=self.arg_shape_dict['rfcn_cls_weight']) arg_params['rfcn_cls_bias'] = mx.nd.zeros(shape=self.arg_shape_dict['rfcn_cls_bias']) arg_params['rfcn_bbox_weight'] = mx.random.normal(0, 0.01, shape=self.arg_shape_dict['rfcn_bbox_weight']) arg_params['rfcn_bbox_bias'] = mx.nd.zeros(shape=self.arg_shape_dict['rfcn_bbox_bias']) ================================================ FILE: dff_rfcn/test.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- import _init_paths import cv2 import argparse import os import sys import time import logging from config.config import config, update_config def parse_args(): parser = argparse.ArgumentParser(description='Test a R-FCN network') # general parser.add_argument('--cfg', help='experiment configure file name', required=True, type=str) args, rest = parser.parse_known_args() update_config(args.cfg) # rcnn parser.add_argument('--vis', help='turn on visualization', action='store_true') parser.add_argument('--ignore_cache', help='ignore cached results boxes', action='store_true') parser.add_argument('--thresh', help='valid detection threshold', default=1e-4, type=float) parser.add_argument('--shuffle', help='shuffle data on visualization', action='store_true') args = parser.parse_args() return args args = parse_args() curr_path = os.path.abspath(os.path.dirname(__file__)) sys.path.insert(0, os.path.join(curr_path, '../external/mxnet', config.MXNET_VERSION)) import mxnet as mx from function.test_rcnn import test_rcnn from utils.create_logger import create_logger def main(): ctx = [mx.gpu(int(i)) for i in config.gpus.split(',')] print args logger, final_output_path = create_logger(config.output_path, args.cfg, config.dataset.test_image_set) test_rcnn(config, config.dataset.dataset, config.dataset.test_image_set, config.dataset.root_path, config.dataset.dataset_path, ctx, os.path.join(final_output_path, '..', '_'.join([iset for iset in config.dataset.image_set.split('+')]), config.TRAIN.model_prefix), config.TEST.test_epoch, args.vis, args.ignore_cache, args.shuffle, config.TEST.HAS_RPN, config.dataset.proposal, args.thresh, logger=logger, output_path=final_output_path) if __name__ == '__main__': main() ================================================ FILE: dff_rfcn/train_end2end.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- import _init_paths import cv2 import time import argparse import logging import pprint import os import sys from config.config import config, update_config def parse_args(): parser = argparse.ArgumentParser(description='Train R-FCN network') # general parser.add_argument('--cfg', help='experiment configure file name', required=True, type=str) args, rest = parser.parse_known_args() # update config update_config(args.cfg) # training parser.add_argument('--frequent', help='frequency of logging', default=config.default.frequent, type=int) args = parser.parse_args() return args args = parse_args() curr_path = os.path.abspath(os.path.dirname(__file__)) sys.path.insert(0, os.path.join(curr_path, '../external/mxnet', config.MXNET_VERSION)) import shutil import numpy as np import mxnet as mx from symbols import * from core import callback, metric from core.loader import AnchorLoader from core.module import MutableModule from utils.create_logger import create_logger from utils.load_data import load_gt_roidb, merge_roidb, filter_roidb from utils.load_model import load_param from utils.PrefetchingIter import PrefetchingIter from utils.lr_scheduler import WarmupMultiFactorScheduler def train_net(args, ctx, pretrained, pretrained_flow, epoch, prefix, begin_epoch, end_epoch, lr, lr_step): logger, final_output_path = create_logger(config.output_path, args.cfg, config.dataset.image_set) prefix = os.path.join(final_output_path, prefix) # load symbol shutil.copy2(os.path.join(curr_path, 'symbols', config.symbol + '.py'), final_output_path) sym_instance = eval(config.symbol + '.' + config.symbol)() sym = sym_instance.get_train_symbol(config) feat_sym = sym.get_internals()['rpn_cls_score_output'] # setup multi-gpu batch_size = len(ctx) input_batch_size = config.TRAIN.BATCH_IMAGES * batch_size # print config pprint.pprint(config) logger.info('training config:{}\n'.format(pprint.pformat(config))) # load dataset and prepare imdb for training image_sets = [iset for iset in config.dataset.image_set.split('+')] roidbs = [load_gt_roidb(config.dataset.dataset, image_set, config.dataset.root_path, config.dataset.dataset_path, flip=config.TRAIN.FLIP) for image_set in image_sets] roidb = merge_roidb(roidbs) roidb = filter_roidb(roidb, config) # load training data train_data = AnchorLoader(feat_sym, roidb, config, batch_size=input_batch_size, shuffle=config.TRAIN.SHUFFLE, ctx=ctx, feat_stride=config.network.RPN_FEAT_STRIDE, anchor_scales=config.network.ANCHOR_SCALES, anchor_ratios=config.network.ANCHOR_RATIOS, aspect_grouping=config.TRAIN.ASPECT_GROUPING, normalize_target=config.network.NORMALIZE_RPN, bbox_mean=config.network.ANCHOR_MEANS, bbox_std=config.network.ANCHOR_STDS) # infer max shape max_data_shape = [('data', (config.TRAIN.BATCH_IMAGES, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES]))), ('data_ref', (config.TRAIN.BATCH_IMAGES, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES]))), ('eq_flag', (1,))] max_data_shape, max_label_shape = train_data.infer_shape(max_data_shape) max_data_shape.append(('gt_boxes', (config.TRAIN.BATCH_IMAGES, 100, 5))) print 'providing maximum shape', max_data_shape, max_label_shape data_shape_dict = dict(train_data.provide_data_single + train_data.provide_label_single) pprint.pprint(data_shape_dict) sym_instance.infer_shape(data_shape_dict) # load and initialize params if config.TRAIN.RESUME: print('continue training from ', begin_epoch) arg_params, aux_params = load_param(prefix, begin_epoch, convert=True) else: arg_params, aux_params = load_param(pretrained, epoch, convert=True) arg_params_flow, aux_params_flow = load_param(pretrained_flow, epoch, convert=True) arg_params.update(arg_params_flow) aux_params.update(aux_params_flow) sym_instance.init_weight(config, arg_params, aux_params) # check parameter shapes sym_instance.check_parameter_shapes(arg_params, aux_params, data_shape_dict) # create solver fixed_param_prefix = config.network.FIXED_PARAMS data_names = [k[0] for k in train_data.provide_data_single] label_names = [k[0] for k in train_data.provide_label_single] mod = MutableModule(sym, data_names=data_names, label_names=label_names, logger=logger, context=ctx, max_data_shapes=[max_data_shape for _ in range(batch_size)], max_label_shapes=[max_label_shape for _ in range(batch_size)], fixed_param_prefix=fixed_param_prefix) if config.TRAIN.RESUME: mod._preload_opt_states = '%s-%04d.states'%(prefix, begin_epoch) # decide training params # metric rpn_eval_metric = metric.RPNAccMetric() rpn_cls_metric = metric.RPNLogLossMetric() rpn_bbox_metric = metric.RPNL1LossMetric() eval_metric = metric.RCNNAccMetric(config) cls_metric = metric.RCNNLogLossMetric(config) bbox_metric = metric.RCNNL1LossMetric(config) eval_metrics = mx.metric.CompositeEvalMetric() # rpn_eval_metric, rpn_cls_metric, rpn_bbox_metric, eval_metric, cls_metric, bbox_metric for child_metric in [rpn_eval_metric, rpn_cls_metric, rpn_bbox_metric, eval_metric, cls_metric, bbox_metric]: eval_metrics.add(child_metric) # callback batch_end_callback = callback.Speedometer(train_data.batch_size, frequent=args.frequent) means = np.tile(np.array(config.TRAIN.BBOX_MEANS), 2 if config.CLASS_AGNOSTIC else config.dataset.NUM_CLASSES) stds = np.tile(np.array(config.TRAIN.BBOX_STDS), 2 if config.CLASS_AGNOSTIC else config.dataset.NUM_CLASSES) epoch_end_callback = [mx.callback.module_checkpoint(mod, prefix, period=1, save_optimizer_states=True), callback.do_checkpoint(prefix, means, stds)] # decide learning rate base_lr = lr lr_factor = config.TRAIN.lr_factor lr_epoch = [float(epoch) for epoch in lr_step.split(',')] lr_epoch_diff = [epoch - begin_epoch for epoch in lr_epoch if epoch > begin_epoch] lr = base_lr * (lr_factor ** (len(lr_epoch) - len(lr_epoch_diff))) lr_iters = [int(epoch * len(roidb) / batch_size) for epoch in lr_epoch_diff] print('lr', lr, 'lr_epoch_diff', lr_epoch_diff, 'lr_iters', lr_iters) lr_scheduler = WarmupMultiFactorScheduler(lr_iters, lr_factor, config.TRAIN.warmup, config.TRAIN.warmup_lr, config.TRAIN.warmup_step) # optimizer optimizer_params = {'momentum': config.TRAIN.momentum, 'wd': config.TRAIN.wd, 'learning_rate': lr, 'lr_scheduler': lr_scheduler, 'rescale_grad': 1.0, 'clip_gradient': None} if not isinstance(train_data, PrefetchingIter): train_data = PrefetchingIter(train_data) # train mod.fit(train_data, eval_metric=eval_metrics, epoch_end_callback=epoch_end_callback, batch_end_callback=batch_end_callback, kvstore=config.default.kvstore, optimizer='sgd', optimizer_params=optimizer_params, arg_params=arg_params, aux_params=aux_params, begin_epoch=begin_epoch, num_epoch=end_epoch) def main(): print('Called with argument:', args) ctx = [mx.gpu(int(i)) for i in config.gpus.split(',')] train_net(args, ctx, config.network.pretrained, config.network.pretrained_flow, config.network.pretrained_epoch, config.TRAIN.model_prefix, config.TRAIN.begin_epoch, config.TRAIN.end_epoch, config.TRAIN.lr, config.TRAIN.lr_step) if __name__ == '__main__': main() ================================================ FILE: experiments/dff_rfcn/cfgs/dff_rfcn_vid_demo.yaml ================================================ --- MXNET_VERSION: "mxnet" output_path: "./output/dff_rfcn/imagenet_vid" gpus: '0' CLASS_AGNOSTIC: true SCALES: - 600 - 1000 default: frequent: 100 kvstore: device network: PIXEL_MEANS: - 103.06 - 115.90 - 123.15 IMAGE_STRIDE: 0 RCNN_FEAT_STRIDE: 16 RPN_FEAT_STRIDE: 16 FIXED_PARAMS: - conv1 - bn_conv1 - res2 - bn2 - gamma - beta ANCHOR_RATIOS: - 0.5 - 1 - 2 ANCHOR_SCALES: - 8 - 16 - 32 ANCHOR_MEANS: - 0.0 - 0.0 - 0.0 - 0.0 ANCHOR_STDS: - 0.1 - 0.1 - 0.4 - 0.4 NORMALIZE_RPN: TRUE NUM_ANCHORS: 9 dataset: NUM_CLASSES: 31 dataset: ImageNetVID dataset_path: "./data/ILSVRC2015" image_set: DET_train_30classes+VID_train_15frames root_path: "./data" test_image_set: VID_val_videos proposal: rpn TRAIN: lr: 0.00025 lr_step: '1.333' warmup: false begin_epoch: 0 end_epoch: 2 model_prefix: 'dff_rfcn_vid' # whether resume training RESUME: false # whether flip image FLIP: true # whether shuffle image SHUFFLE: true # whether use OHEM ENABLE_OHEM: true # size of images for each device, 1 for e2e BATCH_IMAGES: 1 # e2e changes behavior of anchor loader and metric END2END: true # group images with similar aspect ratio ASPECT_GROUPING: true # R-CNN # rcnn rois batch size BATCH_ROIS: -1 BATCH_ROIS_OHEM: 128 # rcnn rois sampling params FG_FRACTION: 0.25 FG_THRESH: 0.5 BG_THRESH_HI: 0.5 BG_THRESH_LO: 0.0 # rcnn bounding box regression params BBOX_REGRESSION_THRESH: 0.5 BBOX_WEIGHTS: - 1.0 - 1.0 - 1.0 - 1.0 # RPN anchor loader # rpn anchors batch size RPN_BATCH_SIZE: 256 # rpn anchors sampling params RPN_FG_FRACTION: 0.5 RPN_POSITIVE_OVERLAP: 0.7 RPN_NEGATIVE_OVERLAP: 0.3 RPN_CLOBBER_POSITIVES: false # rpn bounding box regression params RPN_BBOX_WEIGHTS: - 1.0 - 1.0 - 1.0 - 1.0 RPN_POSITIVE_WEIGHT: -1.0 # used for end2end training # RPN proposal CXX_PROPOSAL: true RPN_NMS_THRESH: 0.7 RPN_PRE_NMS_TOP_N: 6000 RPN_POST_NMS_TOP_N: 300 RPN_MIN_SIZE: 0 # approximate bounding box regression BBOX_NORMALIZATION_PRECOMPUTED: true BBOX_MEANS: - 0.0 - 0.0 - 0.0 - 0.0 BBOX_STDS: - 0.1 - 0.1 - 0.2 - 0.2 TEST: # use rpn to generate proposal HAS_RPN: true # size of images for each device BATCH_IMAGES: 1 # RPN proposal CXX_PROPOSAL: true RPN_NMS_THRESH: 0.7 RPN_PRE_NMS_TOP_N: 6000 RPN_POST_NMS_TOP_N: 300 RPN_MIN_SIZE: 0 # RCNN nms NMS: 0.3 test_epoch: 2 ================================================ FILE: experiments/dff_rfcn/cfgs/resnet_v1_101_flownet_imagenet_vid_rfcn_end2end_ohem.yaml ================================================ --- MXNET_VERSION: "mxnet" output_path: "./output/dff_rfcn/imagenet_vid" symbol: resnet_v1_101_flownet_rfcn gpus: '0,1,2,3' CLASS_AGNOSTIC: true SCALES: - 600 - 1000 default: frequent: 100 kvstore: device network: pretrained: "./model/pretrained_model/resnet_v1_101" pretrained_flow: "./model/pretrained_model/flownet" pretrained_epoch: 0 PIXEL_MEANS: - 103.06 - 115.90 - 123.15 IMAGE_STRIDE: 0 RCNN_FEAT_STRIDE: 16 RPN_FEAT_STRIDE: 16 FIXED_PARAMS: - conv1 - bn_conv1 - res2 - bn2 - gamma - beta ANCHOR_RATIOS: - 0.5 - 1 - 2 ANCHOR_SCALES: - 8 - 16 - 32 ANCHOR_MEANS: - 0.0 - 0.0 - 0.0 - 0.0 ANCHOR_STDS: - 0.1 - 0.1 - 0.4 - 0.4 NORMALIZE_RPN: TRUE NUM_ANCHORS: 9 dataset: NUM_CLASSES: 31 dataset: ImageNetVID dataset_path: "./data/ILSVRC2015" image_set: DET_train_30classes+VID_train_15frames root_path: "./data" test_image_set: VID_val_videos proposal: rpn TRAIN: lr: 0.00025 lr_step: '1.333' warmup: false begin_epoch: 0 end_epoch: 2 model_prefix: 'dff_rfcn_vid' # whether resume training RESUME: false # whether flip image FLIP: true # whether shuffle image SHUFFLE: true # whether use OHEM ENABLE_OHEM: true # size of images for each device, 1 for e2e BATCH_IMAGES: 1 # e2e changes behavior of anchor loader and metric END2END: true # group images with similar aspect ratio ASPECT_GROUPING: true # R-CNN # rcnn rois batch size BATCH_ROIS: -1 BATCH_ROIS_OHEM: 128 # rcnn rois sampling params FG_FRACTION: 0.25 FG_THRESH: 0.5 BG_THRESH_HI: 0.5 BG_THRESH_LO: 0.0 # rcnn bounding box regression params BBOX_REGRESSION_THRESH: 0.5 BBOX_WEIGHTS: - 1.0 - 1.0 - 1.0 - 1.0 # RPN anchor loader # rpn anchors batch size RPN_BATCH_SIZE: 256 # rpn anchors sampling params RPN_FG_FRACTION: 0.5 RPN_POSITIVE_OVERLAP: 0.7 RPN_NEGATIVE_OVERLAP: 0.3 RPN_CLOBBER_POSITIVES: false # rpn bounding box regression params RPN_BBOX_WEIGHTS: - 1.0 - 1.0 - 1.0 - 1.0 RPN_POSITIVE_WEIGHT: -1.0 # used for end2end training # RPN proposal CXX_PROPOSAL: true RPN_NMS_THRESH: 0.7 RPN_PRE_NMS_TOP_N: 6000 RPN_POST_NMS_TOP_N: 300 RPN_MIN_SIZE: 0 # approximate bounding box regression BBOX_NORMALIZATION_PRECOMPUTED: true BBOX_MEANS: - 0.0 - 0.0 - 0.0 - 0.0 BBOX_STDS: - 0.1 - 0.1 - 0.2 - 0.2 TEST: # use rpn to generate proposal HAS_RPN: true # size of images for each device BATCH_IMAGES: 1 # RPN proposal CXX_PROPOSAL: true RPN_NMS_THRESH: 0.7 RPN_PRE_NMS_TOP_N: 6000 RPN_POST_NMS_TOP_N: 300 RPN_MIN_SIZE: 0 # RCNN nms NMS: 0.3 test_epoch: 2 ================================================ FILE: experiments/dff_rfcn/dff_rfcn_end2end_train_test.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong # -------------------------------------------------------- import os import sys os.environ['PYTHONUNBUFFERED'] = '1' os.environ['MXNET_CUDNN_AUTOTUNE_DEFAULT'] = '0' os.environ['MXNET_ENABLE_GPU_P2P'] = '0' this_dir = os.path.dirname(__file__) sys.path.insert(0, os.path.join(this_dir, '..', '..', 'dff_rfcn')) import train_end2end import test if __name__ == "__main__": train_end2end.main() test.main() ================================================ FILE: experiments/dff_rfcn/dff_rfcn_test.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong # -------------------------------------------------------- import os import sys os.environ['PYTHONUNBUFFERED'] = '1' os.environ['MXNET_CUDNN_AUTOTUNE_DEFAULT'] = '0' os.environ['MXNET_ENABLE_GPU_P2P'] = '0' this_dir = os.path.dirname(__file__) sys.path.insert(0, os.path.join(this_dir, '..', '..', 'dff_rfcn')) import test if __name__ == "__main__": test.main() ================================================ FILE: experiments/rfcn/cfgs/resnet_v1_101_imagenet_vid_rfcn_end2end_ohem.yaml ================================================ --- MXNET_VERSION: "mxnet" output_path: "./output/rfcn/imagenet_vid" symbol: resnet_v1_101_rfcn gpus: '0,1,2,3' CLASS_AGNOSTIC: true SCALES: - 600 - 1000 default: frequent: 100 kvstore: device network: pretrained: "./model/pretrained_model/resnet_v1_101" pretrained_epoch: 0 PIXEL_MEANS: - 103.06 - 115.90 - 123.15 IMAGE_STRIDE: 0 RCNN_FEAT_STRIDE: 16 RPN_FEAT_STRIDE: 16 FIXED_PARAMS: - conv1 - bn_conv1 - res2 - bn2 - gamma - beta ANCHOR_RATIOS: - 0.5 - 1 - 2 ANCHOR_SCALES: - 8 - 16 - 32 ANCHOR_MEANS: - 0.0 - 0.0 - 0.0 - 0.0 ANCHOR_STDS: - 0.1 - 0.1 - 0.4 - 0.4 NORMALIZE_RPN: TRUE NUM_ANCHORS: 9 dataset: NUM_CLASSES: 31 dataset: ImageNetVID dataset_path: "./data/ILSVRC2015" image_set: DET_train_30classes+VID_train_15frames root_path: "./data" test_image_set: VID_val_frames proposal: rpn TRAIN: lr: 0.00025 lr_step: '1.333' warmup: false begin_epoch: 0 end_epoch: 2 model_prefix: 'rfcn_vid' # whether resume training RESUME: false # whether flip image FLIP: true # whether shuffle image SHUFFLE: true # whether use OHEM ENABLE_OHEM: true # size of images for each device, 1 for e2e BATCH_IMAGES: 1 # e2e changes behavior of anchor loader and metric END2END: true # group images with similar aspect ratio ASPECT_GROUPING: true # R-CNN # rcnn rois batch size BATCH_ROIS: -1 BATCH_ROIS_OHEM: 128 # rcnn rois sampling params FG_FRACTION: 0.25 FG_THRESH: 0.5 BG_THRESH_HI: 0.5 BG_THRESH_LO: 0.0 # rcnn bounding box regression params BBOX_REGRESSION_THRESH: 0.5 BBOX_WEIGHTS: - 1.0 - 1.0 - 1.0 - 1.0 # RPN anchor loader # rpn anchors batch size RPN_BATCH_SIZE: 256 # rpn anchors sampling params RPN_FG_FRACTION: 0.5 RPN_POSITIVE_OVERLAP: 0.7 RPN_NEGATIVE_OVERLAP: 0.3 RPN_CLOBBER_POSITIVES: false # rpn bounding box regression params RPN_BBOX_WEIGHTS: - 1.0 - 1.0 - 1.0 - 1.0 RPN_POSITIVE_WEIGHT: -1.0 # used for end2end training # RPN proposal CXX_PROPOSAL: true RPN_NMS_THRESH: 0.7 RPN_PRE_NMS_TOP_N: 6000 RPN_POST_NMS_TOP_N: 300 RPN_MIN_SIZE: 0 # approximate bounding box regression BBOX_NORMALIZATION_PRECOMPUTED: true BBOX_MEANS: - 0.0 - 0.0 - 0.0 - 0.0 BBOX_STDS: - 0.1 - 0.1 - 0.2 - 0.2 TEST: # use rpn to generate proposal HAS_RPN: true # size of images for each device BATCH_IMAGES: 1 # RPN proposal CXX_PROPOSAL: true RPN_NMS_THRESH: 0.7 RPN_PRE_NMS_TOP_N: 6000 RPN_POST_NMS_TOP_N: 300 RPN_MIN_SIZE: 0 # RCNN nms NMS: 0.3 test_epoch: 2 ================================================ FILE: experiments/rfcn/cfgs/rfcn_vid_demo.yaml ================================================ --- MXNET_VERSION: "mxnet" output_path: "./output/rfcn/imagenet_vid" gpus: '0' CLASS_AGNOSTIC: true SCALES: - 600 - 1000 default: frequent: 100 kvstore: device network: PIXEL_MEANS: - 103.06 - 115.90 - 123.15 IMAGE_STRIDE: 0 RCNN_FEAT_STRIDE: 16 RPN_FEAT_STRIDE: 16 FIXED_PARAMS: - conv1 - bn_conv1 - res2 - bn2 - gamma - beta ANCHOR_RATIOS: - 0.5 - 1 - 2 ANCHOR_SCALES: - 8 - 16 - 32 ANCHOR_MEANS: - 0.0 - 0.0 - 0.0 - 0.0 ANCHOR_STDS: - 0.1 - 0.1 - 0.4 - 0.4 NORMALIZE_RPN: TRUE NUM_ANCHORS: 9 dataset: NUM_CLASSES: 31 dataset: ImageNetVID dataset_path: "./data/ILSVRC2015" image_set: DET_train_30classes+VID_train_15frames root_path: "./data" test_image_set: VID_val_frames proposal: rpn TRAIN: lr: 0.00025 lr_step: '1.333' warmup: false begin_epoch: 0 end_epoch: 2 model_prefix: 'rfcn_vid' # whether resume training RESUME: false # whether flip image FLIP: true # whether shuffle image SHUFFLE: true # whether use OHEM ENABLE_OHEM: true # size of images for each device, 1 for e2e BATCH_IMAGES: 1 # e2e changes behavior of anchor loader and metric END2END: true # group images with similar aspect ratio ASPECT_GROUPING: true # R-CNN # rcnn rois batch size BATCH_ROIS: -1 BATCH_ROIS_OHEM: 128 # rcnn rois sampling params FG_FRACTION: 0.25 FG_THRESH: 0.5 BG_THRESH_HI: 0.5 BG_THRESH_LO: 0.0 # rcnn bounding box regression params BBOX_REGRESSION_THRESH: 0.5 BBOX_WEIGHTS: - 1.0 - 1.0 - 1.0 - 1.0 # RPN anchor loader # rpn anchors batch size RPN_BATCH_SIZE: 256 # rpn anchors sampling params RPN_FG_FRACTION: 0.5 RPN_POSITIVE_OVERLAP: 0.7 RPN_NEGATIVE_OVERLAP: 0.3 RPN_CLOBBER_POSITIVES: false # rpn bounding box regression params RPN_BBOX_WEIGHTS: - 1.0 - 1.0 - 1.0 - 1.0 RPN_POSITIVE_WEIGHT: -1.0 # used for end2end training # RPN proposal CXX_PROPOSAL: true RPN_NMS_THRESH: 0.7 RPN_PRE_NMS_TOP_N: 6000 RPN_POST_NMS_TOP_N: 300 RPN_MIN_SIZE: 0 # approximate bounding box regression BBOX_NORMALIZATION_PRECOMPUTED: true BBOX_MEANS: - 0.0 - 0.0 - 0.0 - 0.0 BBOX_STDS: - 0.1 - 0.1 - 0.2 - 0.2 TEST: # use rpn to generate proposal HAS_RPN: true # size of images for each device BATCH_IMAGES: 1 # RPN proposal CXX_PROPOSAL: true RPN_NMS_THRESH: 0.7 RPN_PRE_NMS_TOP_N: 6000 RPN_POST_NMS_TOP_N: 300 RPN_MIN_SIZE: 0 # RCNN nms NMS: 0.3 test_epoch: 2 ================================================ FILE: experiments/rfcn/rfcn_end2end_train_test.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong # -------------------------------------------------------- import os import sys os.environ['PYTHONUNBUFFERED'] = '1' os.environ['MXNET_CUDNN_AUTOTUNE_DEFAULT'] = '0' os.environ['MXNET_ENABLE_GPU_P2P'] = '0' this_dir = os.path.dirname(__file__) sys.path.insert(0, os.path.join(this_dir, '..', '..', 'rfcn')) import train_end2end import test if __name__ == "__main__": train_end2end.main() test.main() ================================================ FILE: experiments/rfcn/rfcn_test.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong # -------------------------------------------------------- import os import sys os.environ['PYTHONUNBUFFERED'] = '1' os.environ['MXNET_CUDNN_AUTOTUNE_DEFAULT'] = '0' os.environ['MXNET_ENABLE_GPU_P2P'] = '0' this_dir = os.path.dirname(__file__) sys.path.insert(0, os.path.join(this_dir, '..', '..', 'rfcn')) import test if __name__ == "__main__": test.main() ================================================ FILE: green2.py ================================================ # _*_ coding: utf-8 _*_ import datetime import os import random from random import choice comments = [ 'Update ReactFiberScheduler', 'Update CHANGELOG for 16.7', 'v16.7.0', 'Move SchedulerFeatureFlags fork to src directory to fix lint', 'Make scheduler debugging feature flag static', 'Create separate SchedulerFeatureFlags instead of using ReactFeatureFl…', 'Memoize promise listeners to prevent exponential growth', 'Removed Fabric-specific feature flag files and updated Rollup to use', 'Updated version incrementing suggestion in release script based on te…', 'Implement pauseExecution, continueExecution, dumpQueue for Scheduler', 'Added ErrorBoundary tests for useEffect and useLayoutEffect', 'Enable hooks in fabric', 'Tweaked wording for v8 "performance cliff" issue', 'Automated fixture tests', 'Removed unnecessary externals from Jest bundles', 'Fixed scheduler setTimeout fallback', 'Fix bug in cloneHook', '[Fire] Add initial build infrastructure', 'Adding isMemo check to react-is package', 'Remove useMutationEffect', 'Add a null type test for memo', 'Minified Deep Learn ', 'Deep net learn', 'update credits', 'update url', 'Update rnnrbm.py', 'Update segmentation tutorials and .gitignore.', 'Preload data for 1D cortical dataset', 'Results for unet', 'added per class jaccard', 'Formatting', 'Remove files that are not used any more', 'More small fixes', 'data augmentation fixed', 'remove loading pretrained weights', 'index updated with new links', 'removed files .pyc', 'small change', 'import conv2DDDNlayer', 'fix input dim and details', 'fix dataset loader for em, name for polyps', 'adde training loop link', 'small changes', 'relative paths', 'with BN', 'update from pascal comment', 'polyps results image', 'acknowledgements', 'added code', 'first commit for fcn1D segmentations', 'deleted files', 'figure 4 description', 'explanation ground truth vs predicted segmentation', 'FCN description fixed', 'cortical layers imagse' ] def modify(): file = open('zero.md', 'r') flag = file.readline() == '0' file.close() file = open('zero.md', 'w+') if flag: file.write('1') else: file.write('0') file.close() def commit(): cm = 'git commit -a -m "' + choice(comments) + '"' print(cm) os.system(cm) def set_sys_time(day, month, year): os.system('date %02d%02d1720%04d' % (month, day, year)) def trick_commit(year, month, day): set_sys_time(year, month, day) modify() commit() def daily_commit(start_date, end_date): count = 0 while (count < 2): for i in range((end_date - start_date).days - 5): if (i%random.randint(1,2)): cur_date = start_date + datetime.timedelta(days=(i+random.randint(1,5))) trick_commit(cur_date.day, cur_date.month, cur_date.year) count = count + 1 if __name__ == '__main__': daily_commit(datetime.date(2015, 2, 21), datetime.date(2018, 12, 19)) ================================================ FILE: init.bat ================================================ cd /d %~dp0 mkdir .\output mkdir .\external\mxnet mkdir .\model\pretrained_model pause cd lib\bbox python setup_windows.py build_ext --inplace cd ..\nms python setup_windows.py build_ext --inplace python setup_windows_cuda.py build_ext --inplace cd ..\.. pause ================================================ FILE: init.sh ================================================ #!/bin/bash mkdir -p ./output mkdir -p ./external/mxnet mkdir -p ./model/pretrained_model cd lib/bbox python setup_linux.py build_ext --inplace cd ../nms python setup_linux.py build_ext --inplace cd ../.. ================================================ FILE: lib/Makefile ================================================ all: cd nms/; python setup.py build_ext --inplace; rm -rf build; cd ../../ cd bbox/; python setup.py build_ext --inplace; rm -rf build; cd ../../ cd dataset/pycocotools/; python setup.py build_ext --inplace; rm -rf build; cd ../../ clean: cd nms/; rm *.so *.c *.cpp; cd ../../ cd bbox/; rm *.so *.c *.cpp; cd ../../ cd dataset/pycocotools/; rm *.so; cd ../../ ================================================ FILE: lib/__init__.py ================================================ ================================================ FILE: lib/bbox/.gitignore ================================================ *.c *.cpp ================================================ FILE: lib/bbox/__init__.py ================================================ ================================================ FILE: lib/bbox/bbox.pyx ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Sergey Karayev # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # py-faster-rcnn # Copyright (c) 2015 Microsoft # Licence under The MIT License # https://github.com/rbgirshick/py-faster-rcnn # -------------------------------------------------------- cimport cython import numpy as np cimport numpy as np DTYPE = np.float ctypedef np.float_t DTYPE_t def bbox_overlaps_cython( np.ndarray[DTYPE_t, ndim=2] boxes, np.ndarray[DTYPE_t, ndim=2] query_boxes): """ Parameters ---------- boxes: (N, 4) ndarray of float query_boxes: (K, 4) ndarray of float Returns ------- overlaps: (N, K) ndarray of overlap between boxes and query_boxes """ cdef unsigned int N = boxes.shape[0] cdef unsigned int K = query_boxes.shape[0] cdef np.ndarray[DTYPE_t, ndim=2] overlaps = np.zeros((N, K), dtype=DTYPE) cdef DTYPE_t iw, ih, box_area cdef DTYPE_t ua cdef unsigned int k, n for k in range(K): box_area = ( (query_boxes[k, 2] - query_boxes[k, 0] + 1) * (query_boxes[k, 3] - query_boxes[k, 1] + 1) ) for n in range(N): iw = ( min(boxes[n, 2], query_boxes[k, 2]) - max(boxes[n, 0], query_boxes[k, 0]) + 1 ) if iw > 0: ih = ( min(boxes[n, 3], query_boxes[k, 3]) - max(boxes[n, 1], query_boxes[k, 1]) + 1 ) if ih > 0: ua = float( (boxes[n, 2] - boxes[n, 0] + 1) * (boxes[n, 3] - boxes[n, 1] + 1) + box_area - iw * ih ) overlaps[n, k] = iw * ih / ua return overlaps ================================================ FILE: lib/bbox/bbox_regression.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # py-faster-rcnn # Copyright (c) 2016 by Contributors # Licence under The MIT License # https://github.com/rbgirshick/py-faster-rcnn # -------------------------------------------------------- """ This file has functions about generating bounding box regression targets """ import numpy as np from bbox_transform import bbox_overlaps, bbox_transform def compute_bbox_regression_targets(rois, overlaps, labels, cfg): """ given rois, overlaps, gt labels, compute bounding box regression targets :param rois: roidb[i]['boxes'] k * 4 :param overlaps: roidb[i]['max_overlaps'] k * 1 :param labels: roidb[i]['max_classes'] k * 1 :return: targets[i][class, dx, dy, dw, dh] k * 5 """ # Ensure ROIs are floats rois = rois.astype(np.float, copy=False) # Sanity check if len(rois) != len(overlaps): print 'bbox regression: this should not happen' # Indices of ground-truth ROIs gt_inds = np.where(overlaps == 1)[0] if len(gt_inds) == 0: print 'something wrong : zero ground truth rois' # Indices of examples for which we try to make predictions ex_inds = np.where(overlaps >= cfg.TRAIN.BBOX_REGRESSION_THRESH)[0] # Get IoU overlap between each ex ROI and gt ROI ex_gt_overlaps = bbox_overlaps(rois[ex_inds, :], rois[gt_inds, :]) # Find which gt ROI each ex ROI has max overlap with: # this will be the ex ROI's gt target gt_assignment = ex_gt_overlaps.argmax(axis=1) gt_rois = rois[gt_inds[gt_assignment], :] ex_rois = rois[ex_inds, :] targets = np.zeros((rois.shape[0], 5), dtype=np.float32) targets[ex_inds, 0] = labels[ex_inds] targets[ex_inds, 1:] = bbox_transform(ex_rois, gt_rois) return targets def add_bbox_regression_targets(roidb, cfg): """ given roidb, add ['bbox_targets'] and normalize bounding box regression targets :param roidb: roidb to be processed. must have gone through imdb.prepare_roidb :return: means, std variances of targets """ print 'add bounding box regression targets' assert len(roidb) > 0 assert 'max_classes' in roidb[0] num_images = len(roidb) num_classes = 2 if cfg.CLASS_AGNOSTIC else roidb[0]['gt_overlaps'].shape[1] for im_i in range(num_images): rois = roidb[im_i]['boxes'] max_overlaps = roidb[im_i]['max_overlaps'] max_classes = roidb[im_i]['max_classes'] roidb[im_i]['bbox_targets'] = compute_bbox_regression_targets(rois, max_overlaps, max_classes, cfg) if cfg.TRAIN.BBOX_NORMALIZATION_PRECOMPUTED: # use fixed / precomputed means and stds instead of empirical values means = np.tile(np.array(cfg.TRAIN.BBOX_MEANS), (num_classes, 1)) stds = np.tile(np.array(cfg.TRAIN.BBOX_STDS), (num_classes, 1)) else: # compute mean, std values class_counts = np.zeros((num_classes, 1)) + 1e-14 sums = np.zeros((num_classes, 4)) squared_sums = np.zeros((num_classes, 4)) for im_i in range(num_images): targets = roidb[im_i]['bbox_targets'] for cls in range(1, num_classes): cls_indexes = np.where(targets[:, 0] > 0)[0] if cfg.CLASS_AGNOSTIC else np.where(targets[:, 0] == cls)[0] if cls_indexes.size > 0: class_counts[cls] += cls_indexes.size sums[cls, :] += targets[cls_indexes, 1:].sum(axis=0) squared_sums[cls, :] += (targets[cls_indexes, 1:] ** 2).sum(axis=0) means = sums / class_counts # var(x) = E(x^2) - E(x)^2 stds = np.sqrt(squared_sums / class_counts - means ** 2) print 'bbox target means:' print means print means[1:, :].mean(axis=0) # ignore bg class print 'bbox target stdevs:' print stds print stds[1:, :].mean(axis=0) # ignore bg class # normalized targets for im_i in range(num_images): targets = roidb[im_i]['bbox_targets'] for cls in range(1, num_classes): cls_indexes = np.where(targets[:, 0] > 0) if cfg.CLASS_AGNOSTIC else np.where(targets[:, 0] == cls)[0] roidb[im_i]['bbox_targets'][cls_indexes, 1:] -= means[cls, :] roidb[im_i]['bbox_targets'][cls_indexes, 1:] /= stds[cls, :] return means.ravel(), stds.ravel() def expand_bbox_regression_targets(bbox_targets_data, num_classes, cfg): """ expand from 5 to 4 * num_classes; only the right class has non-zero bbox regression targets :param bbox_targets_data: [k * 5] :param num_classes: number of classes :return: bbox target processed [k * 4 num_classes] bbox_weights ! only foreground boxes have bbox regression computation! """ classes = bbox_targets_data[:, 0] if cfg.CLASS_AGNOSTIC: num_classes = 2 bbox_targets = np.zeros((classes.size, 4 * num_classes), dtype=np.float32) bbox_weights = np.zeros(bbox_targets.shape, dtype=np.float32) indexes = np.where(classes > 0)[0] for index in indexes: cls = classes[index] start = int(4 * 1 if cls > 0 else 0) if cfg.CLASS_AGNOSTIC else int(4 * cls) end = start + 4 bbox_targets[index, start:end] = bbox_targets_data[index, 1:] bbox_weights[index, start:end] = cfg.TRAIN.BBOX_WEIGHTS return bbox_targets, bbox_weights ================================================ FILE: lib/bbox/bbox_transform.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # py-faster-rcnn # Copyright (c) 2016 by Contributors # Licence under The MIT License # https://github.com/rbgirshick/py-faster-rcnn # -------------------------------------------------------- import numpy as np from bbox import bbox_overlaps_cython def bbox_overlaps(boxes, query_boxes): return bbox_overlaps_cython(boxes, query_boxes) def bbox_overlaps_py(boxes, query_boxes): """ determine overlaps between boxes and query_boxes :param boxes: n * 4 bounding boxes :param query_boxes: k * 4 bounding boxes :return: overlaps: n * k overlaps """ n_ = boxes.shape[0] k_ = query_boxes.shape[0] overlaps = np.zeros((n_, k_), dtype=np.float) for k in range(k_): query_box_area = (query_boxes[k, 2] - query_boxes[k, 0] + 1) * (query_boxes[k, 3] - query_boxes[k, 1] + 1) for n in range(n_): iw = min(boxes[n, 2], query_boxes[k, 2]) - max(boxes[n, 0], query_boxes[k, 0]) + 1 if iw > 0: ih = min(boxes[n, 3], query_boxes[k, 3]) - max(boxes[n, 1], query_boxes[k, 1]) + 1 if ih > 0: box_area = (boxes[n, 2] - boxes[n, 0] + 1) * (boxes[n, 3] - boxes[n, 1] + 1) all_area = float(box_area + query_box_area - iw * ih) overlaps[n, k] = iw * ih / all_area return overlaps def clip_boxes(boxes, im_shape): """ Clip boxes to image boundaries. :param boxes: [N, 4* num_classes] :param im_shape: tuple of 2 :return: [N, 4* num_classes] """ # x1 >= 0 boxes[:, 0::4] = np.maximum(np.minimum(boxes[:, 0::4], im_shape[1] - 1), 0) # y1 >= 0 boxes[:, 1::4] = np.maximum(np.minimum(boxes[:, 1::4], im_shape[0] - 1), 0) # x2 < im_shape[1] boxes[:, 2::4] = np.maximum(np.minimum(boxes[:, 2::4], im_shape[1] - 1), 0) # y2 < im_shape[0] boxes[:, 3::4] = np.maximum(np.minimum(boxes[:, 3::4], im_shape[0] - 1), 0) return boxes def filter_boxes(boxes, min_size): """ filter small boxes. :param boxes: [N, 4* num_classes] :param min_size: :return: keep: """ ws = boxes[:, 2] - boxes[:, 0] + 1 hs = boxes[:, 3] - boxes[:, 1] + 1 keep = np.where((ws >= min_size) & (hs >= min_size))[0] return keep def nonlinear_transform(ex_rois, gt_rois): """ compute bounding box regression targets from ex_rois to gt_rois :param ex_rois: [N, 4] :param gt_rois: [N, 4] :return: [N, 4] """ assert ex_rois.shape[0] == gt_rois.shape[0], 'inconsistent rois number' ex_widths = ex_rois[:, 2] - ex_rois[:, 0] + 1.0 ex_heights = ex_rois[:, 3] - ex_rois[:, 1] + 1.0 ex_ctr_x = ex_rois[:, 0] + 0.5 * (ex_widths - 1.0) ex_ctr_y = ex_rois[:, 1] + 0.5 * (ex_heights - 1.0) gt_widths = gt_rois[:, 2] - gt_rois[:, 0] + 1.0 gt_heights = gt_rois[:, 3] - gt_rois[:, 1] + 1.0 gt_ctr_x = gt_rois[:, 0] + 0.5 * (gt_widths - 1.0) gt_ctr_y = gt_rois[:, 1] + 0.5 * (gt_heights - 1.0) targets_dx = (gt_ctr_x - ex_ctr_x) / (ex_widths + 1e-14) targets_dy = (gt_ctr_y - ex_ctr_y) / (ex_heights + 1e-14) targets_dw = np.log(gt_widths / ex_widths) targets_dh = np.log(gt_heights / ex_heights) targets = np.vstack( (targets_dx, targets_dy, targets_dw, targets_dh)).transpose() return targets def nonlinear_pred(boxes, box_deltas): """ Transform the set of class-agnostic boxes into class-specific boxes by applying the predicted offsets (box_deltas) :param boxes: !important [N 4] :param box_deltas: [N, 4 * num_classes] :return: [N 4 * num_classes] """ if boxes.shape[0] == 0: return np.zeros((0, box_deltas.shape[1])) boxes = boxes.astype(np.float, copy=False) widths = boxes[:, 2] - boxes[:, 0] + 1.0 heights = boxes[:, 3] - boxes[:, 1] + 1.0 ctr_x = boxes[:, 0] + 0.5 * (widths - 1.0) ctr_y = boxes[:, 1] + 0.5 * (heights - 1.0) dx = box_deltas[:, 0::4] dy = box_deltas[:, 1::4] dw = box_deltas[:, 2::4] dh = box_deltas[:, 3::4] pred_ctr_x = dx * widths[:, np.newaxis] + ctr_x[:, np.newaxis] pred_ctr_y = dy * heights[:, np.newaxis] + ctr_y[:, np.newaxis] pred_w = np.exp(dw) * widths[:, np.newaxis] pred_h = np.exp(dh) * heights[:, np.newaxis] pred_boxes = np.zeros(box_deltas.shape) # x1 pred_boxes[:, 0::4] = pred_ctr_x - 0.5 * (pred_w - 1.0) # y1 pred_boxes[:, 1::4] = pred_ctr_y - 0.5 * (pred_h - 1.0) # x2 pred_boxes[:, 2::4] = pred_ctr_x + 0.5 * (pred_w - 1.0) # y2 pred_boxes[:, 3::4] = pred_ctr_y + 0.5 * (pred_h - 1.0) return pred_boxes def iou_transform(ex_rois, gt_rois): """ return bbox targets, IoU loss uses gt_rois as gt """ assert ex_rois.shape[0] == gt_rois.shape[0], 'inconsistent rois number' return gt_rois def iou_pred(boxes, box_deltas): """ Transform the set of class-agnostic boxes into class-specific boxes by applying the predicted offsets (box_deltas) :param boxes: !important [N 4] :param box_deltas: [N, 4 * num_classes] :return: [N 4 * num_classes] """ if boxes.shape[0] == 0: return np.zeros((0, box_deltas.shape[1])) boxes = boxes.astype(np.float, copy=False) x1 = boxes[:, 0] y1 = boxes[:, 1] x2 = boxes[:, 2] y2 = boxes[:, 3] dx1 = box_deltas[:, 0::4] dy1 = box_deltas[:, 1::4] dx2 = box_deltas[:, 2::4] dy2 = box_deltas[:, 3::4] pred_boxes = np.zeros(box_deltas.shape) # x1 pred_boxes[:, 0::4] = dx1 + x1[:, np.newaxis] # y1 pred_boxes[:, 1::4] = dy1 + y1[:, np.newaxis] # x2 pred_boxes[:, 2::4] = dx2 + x2[:, np.newaxis] # y2 pred_boxes[:, 3::4] = dy2 + y2[:, np.newaxis] return pred_boxes # define bbox_transform and bbox_pred bbox_transform = nonlinear_transform bbox_pred = nonlinear_pred ================================================ FILE: lib/bbox/setup_linux.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # py-faster-rcnn # Copyright (c) 2015 Microsoft # Licence under The MIT License # https://github.com/rbgirshick/py-faster-rcnn # -------------------------------------------------------- import os from os.path import join as pjoin from setuptools import setup from distutils.extension import Extension from Cython.Distutils import build_ext import numpy as np # Obtain the numpy include directory. This logic works across numpy versions. try: numpy_include = np.get_include() except AttributeError: numpy_include = np.get_numpy_include() def customize_compiler_for_nvcc(self): """inject deep into distutils to customize how the dispatch to gcc/nvcc works. If you subclass UnixCCompiler, it's not trivial to get your subclass injected in, and still have the right customizations (i.e. distutils.sysconfig.customize_compiler) run on it. So instead of going the OO route, I have this. Note, it's kindof like a wierd functional subclassing going on.""" # tell the compiler it can processes .cu self.src_extensions.append('.cu') # save references to the default compiler_so and _comple methods default_compiler_so = self.compiler_so super = self._compile # now redefine the _compile method. This gets executed for each # object but distutils doesn't have the ability to change compilers # based on source extension: we add it. def _compile(obj, src, ext, cc_args, extra_postargs, pp_opts): if os.path.splitext(src)[1] == '.cu': # use the cuda for .cu files self.set_executable('compiler_so', CUDA['nvcc']) # use only a subset of the extra_postargs, which are 1-1 translated # from the extra_compile_args in the Extension class postargs = extra_postargs['nvcc'] else: postargs = extra_postargs['gcc'] super(obj, src, ext, cc_args, postargs, pp_opts) # reset the default compiler_so, which we might have changed for cuda self.compiler_so = default_compiler_so # inject our redefined _compile method into the class self._compile = _compile # run the customize_compiler class custom_build_ext(build_ext): def build_extensions(self): customize_compiler_for_nvcc(self.compiler) build_ext.build_extensions(self) ext_modules = [ Extension( "bbox", ["bbox.pyx"], extra_compile_args={'gcc': ["-Wno-cpp", "-Wno-unused-function"]}, include_dirs=[numpy_include] ), ] setup( name='bbox_cython', ext_modules=ext_modules, # inject our custom trigger cmdclass={'build_ext': custom_build_ext}, ) ================================================ FILE: lib/bbox/setup_windows.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # py-faster-rcnn # Copyright (c) 2015 Microsoft # Licence under The MIT License # https://github.com/rbgirshick/py-faster-rcnn # -------------------------------------------------------- import numpy as np import os from os.path import join as pjoin #from distutils.core import setup from setuptools import setup from distutils.extension import Extension from Cython.Distutils import build_ext import subprocess #change for windows, by MrX nvcc_bin = 'nvcc.exe' lib_dir = 'lib/x64' import distutils.msvc9compiler distutils.msvc9compiler.VERSION = 14.0 # Obtain the numpy include directory. This logic works across numpy versions. try: numpy_include = np.get_include() except AttributeError: numpy_include = np.get_numpy_include() ext_modules = [ # unix _compile: obj, src, ext, cc_args, extra_postargs, pp_opts Extension( "bbox", sources=["bbox.pyx"], extra_compile_args={}, include_dirs = [numpy_include] ), ] setup( name='fast_rcnn', ext_modules=ext_modules, # inject our custom trigger cmdclass={'build_ext': build_ext}, ) ================================================ FILE: lib/dataset/__init__.py ================================================ from imdb import IMDB from imagenet_vid import ImageNetVID ================================================ FILE: lib/dataset/ds_utils.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong # -------------------------------------------------------- import numpy as np def unique_boxes(boxes, scale=1.0): """ return indices of unique boxes """ v = np.array([1, 1e3, 1e6, 1e9]) hashes = np.round(boxes * scale).dot(v) _, index = np.unique(hashes, return_index=True) return np.sort(index) def filter_small_boxes(boxes, min_size): w = boxes[:, 2] - boxes[:, 0] h = boxes[:, 3] - boxes[:, 1] keep = np.where((w >= min_size) & (h > min_size))[0] return keep ================================================ FILE: lib/dataset/imagenet_vid.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Xizhou Zhu # -------------------------------------------------------- """ ImageNet VID database This class loads ground truth notations from standard ImageNet VID XML data formats and transform them into IMDB format. Selective search is used for proposals, see roidb function. Results are written as the ImageNet VID format. Evaluation is based on mAP criterion. """ import cPickle import cv2 import os import numpy as np from imdb import IMDB from imagenet_vid_eval import vid_eval from ds_utils import unique_boxes, filter_small_boxes class ImageNetVID(IMDB): def __init__(self, image_set, root_path, dataset_path, result_path=None): """ fill basic information to initialize imdb """ det_vid = image_set.split('_')[0] super(ImageNetVID, self).__init__('ImageNetVID', image_set, root_path, dataset_path, result_path) # set self.name self.det_vid = det_vid self.root_path = root_path self.data_path = dataset_path self.classes = ['__background__', # always index 0 'airplane', 'antelope', 'bear', 'bicycle', 'bird', 'bus', 'car', 'cattle', 'dog', 'domestic_cat', 'elephant', 'fox', 'giant_panda', 'hamster', 'horse', 'lion', 'lizard', 'monkey', 'motorcycle', 'rabbit', 'red_panda', 'sheep', 'snake', 'squirrel', 'tiger', 'train', 'turtle', 'watercraft', 'whale', 'zebra'] self.classes_map = ['__background__', # always index 0 'n02691156', 'n02419796', 'n02131653', 'n02834778', 'n01503061', 'n02924116', 'n02958343', 'n02402425', 'n02084071', 'n02121808', 'n02503517', 'n02118333', 'n02510455', 'n02342885', 'n02374451', 'n02129165', 'n01674464', 'n02484322', 'n03790512', 'n02324045', 'n02509815', 'n02411705', 'n01726692', 'n02355227', 'n02129604', 'n04468005', 'n01662784', 'n04530566', 'n02062744', 'n02391049'] self.num_classes = len(self.classes) self.load_image_set_index() self.num_images = len(self.image_set_index) print 'num_images', self.num_images def load_image_set_index(self): """ find out which indexes correspond to given image set (train or val) :return: """ image_set_index_file = os.path.join(self.data_path, 'ImageSets', self.image_set + '.txt') assert os.path.exists(image_set_index_file), 'Path does not exist: {}'.format(image_set_index_file) with open(image_set_index_file) as f: lines = [x.strip().split(' ') for x in f.readlines()] if len(lines[0]) == 2: self.image_set_index = [x[0] for x in lines] self.frame_id = [int(x[1]) for x in lines] else: self.image_set_index = ['%s/%06d' % (x[0], int(x[2])) for x in lines] self.pattern = [x[0]+'/%06d' for x in lines] self.frame_id = [int(x[1]) for x in lines] self.frame_seg_id = [int(x[2]) for x in lines] self.frame_seg_len = [int(x[3]) for x in lines] # return image_set_index, frame_id def image_path_from_index(self, index): """ given image index, find out full path :param index: index of a specific image :return: full path of this image """ if self.det_vid == 'DET': image_file = os.path.join(self.data_path, 'Data', 'DET', index + '.JPEG') else: image_file = os.path.join(self.data_path, 'Data', 'VID', index + '.JPEG') # assert os.path.exists(image_file), 'Path does not exist: {}'.format(image_file) return image_file def gt_roidb(self): """ return ground truth image regions database :return: imdb[image_index]['boxes', 'gt_classes', 'gt_overlaps', 'flipped'] """ cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl') if os.path.exists(cache_file): with open(cache_file, 'rb') as fid: roidb = cPickle.load(fid) print '{} gt roidb loaded from {}'.format(self.name, cache_file) return roidb gt_roidb = [self.load_vid_annotation(index) for index in range(0, len(self.image_set_index))] with open(cache_file, 'wb') as fid: cPickle.dump(gt_roidb, fid, cPickle.HIGHEST_PROTOCOL) print 'wrote gt roidb to {}'.format(cache_file) return gt_roidb def load_vid_annotation(self, iindex): """ for a given index, load image and bounding boxes info from XML file :param index: index of a specific image :return: record['boxes', 'gt_classes', 'gt_overlaps', 'flipped'] """ index = self.image_set_index[iindex] import xml.etree.ElementTree as ET roi_rec = dict() roi_rec['image'] = self.image_path_from_index(index) roi_rec['frame_id'] = self.frame_id[iindex] if hasattr(self,'frame_seg_id'): roi_rec['pattern'] = self.image_path_from_index(self.pattern[iindex]) roi_rec['frame_seg_id'] = self.frame_seg_id[iindex] roi_rec['frame_seg_len'] = self.frame_seg_len[iindex] if self.det_vid == 'DET': filename = os.path.join(self.data_path, 'Annotations', 'DET', index + '.xml') else: filename = os.path.join(self.data_path, 'Annotations', 'VID', index + '.xml') tree = ET.parse(filename) size = tree.find('size') roi_rec['height'] = float(size.find('height').text) roi_rec['width'] = float(size.find('width').text) #im_size = cv2.imread(roi_rec['image'], cv2.IMREAD_COLOR|cv2.IMREAD_IGNORE_ORIENTATION).shape #assert im_size[0] == roi_rec['height'] and im_size[1] == roi_rec['width'] objs = tree.findall('object') num_objs = len(objs) boxes = np.zeros((num_objs, 4), dtype=np.uint16) gt_classes = np.zeros((num_objs), dtype=np.int32) overlaps = np.zeros((num_objs, self.num_classes), dtype=np.float32) valid_objs = np.zeros((num_objs), dtype=np.bool) class_to_index = dict(zip(self.classes_map, range(self.num_classes))) # Load object bounding boxes into a data frame. for ix, obj in enumerate(objs): bbox = obj.find('bndbox') # Make pixel indexes 0-based x1 = np.maximum(float(bbox.find('xmin').text), 0) y1 = np.maximum(float(bbox.find('ymin').text), 0) x2 = np.minimum(float(bbox.find('xmax').text), roi_rec['width']-1) y2 = np.minimum(float(bbox.find('ymax').text), roi_rec['height']-1) if not class_to_index.has_key(obj.find('name').text): continue valid_objs[ix] = True cls = class_to_index[obj.find('name').text.lower().strip()] boxes[ix, :] = [x1, y1, x2, y2] gt_classes[ix] = cls overlaps[ix, cls] = 1.0 boxes = boxes[valid_objs, :] gt_classes = gt_classes[valid_objs] overlaps = overlaps[valid_objs, :] assert (boxes[:, 2] >= boxes[:, 0]).all() roi_rec.update({'boxes': boxes, 'gt_classes': gt_classes, 'gt_overlaps': overlaps, 'max_classes': overlaps.argmax(axis=1), 'max_overlaps': overlaps.max(axis=1), 'flipped': False}) return roi_rec ################################################################################################### def evaluate_detections(self, detections): """ top level evaluations :param detections: result matrix, [bbox, confidence] :return: None """ # make all these folders for results result_dir = os.path.join(self.result_path, 'results') if not os.path.exists(result_dir): os.mkdir(result_dir) self.write_vid_results(detections) info = self.do_python_eval() return info def evaluate_detections_multiprocess(self, detections): """ top level evaluations :param detections: result matrix, [bbox, confidence] :return: None """ # make all these folders for results result_dir = os.path.join(self.result_path, 'results') if not os.path.exists(result_dir): os.mkdir(result_dir) self.write_vid_results_multiprocess(detections) info = self.do_python_eval_gen() return info def get_result_file_template(self): """ :return: a string template """ res_file_folder = os.path.join(self.result_path, 'results') filename = 'det_' + self.image_set + '_{:s}.txt' path = os.path.join(res_file_folder, filename) return path def write_vid_results(self, all_boxes): """ write results files in pascal devkit path :param all_boxes: boxes to be processed [bbox, confidence] :return: None """ print 'Writing {} ImageNetVID results file'.format('all') filename = self.get_result_file_template().format('all') with open(filename, 'wt') as f: for im_ind, index in enumerate(self.image_set_index): for cls_ind, cls in enumerate(self.classes): if cls == '__background__': continue dets = all_boxes[cls_ind][im_ind] if len(dets) == 0: continue # the imagenet expects 0-based indices for k in range(dets.shape[0]): f.write('{:d} {:d} {:.4f} {:.2f} {:.2f} {:.2f} {:.2f}\n'. format(self.frame_id[im_ind], cls_ind, dets[k, -1], dets[k, 0], dets[k, 1], dets[k, 2], dets[k, 3])) def write_vid_results_multiprocess(self, detections): """ write results files in pascal devkit path :param all_boxes: boxes to be processed [bbox, confidence] :return: None """ print 'Writing {} ImageNetVID results file'.format('all') filename = self.get_result_file_template().format('all') with open(filename, 'wt') as f: for detection in detections: all_boxes = detection[0] frame_ids = detection[1] for im_ind in range(len(frame_ids)): for cls_ind, cls in enumerate(self.classes): if cls == '__background__': continue dets = all_boxes[cls_ind][im_ind] if len(dets) == 0: continue # the imagenet expects 0-based indices for k in range(dets.shape[0]): f.write('{:d} {:d} {:.4f} {:.2f} {:.2f} {:.2f} {:.2f}\n'. format(frame_ids[im_ind], cls_ind, dets[k, -1], dets[k, 0], dets[k, 1], dets[k, 2], dets[k, 3])) def do_python_eval(self): """ python evaluation wrapper :return: info_str """ info_str = '' annopath = os.path.join(self.data_path, 'Annotations', '{0!s}.xml') imageset_file = os.path.join(self.data_path, 'ImageSets', self.image_set + '.txt') annocache = os.path.join(self.cache_path, self.name + '_annotations.pkl') filename = self.get_result_file_template().format('all') ap = vid_eval(filename, annopath, imageset_file, self.classes_map, annocache, ovthresh=0.5) for cls_ind, cls in enumerate(self.classes): if cls == '__background__': continue print('AP for {} = {:.4f}'.format(cls, ap[cls_ind-1])) info_str += 'AP for {} = {:.4f}\n'.format(cls, ap[cls_ind-1]) print('Mean AP@0.5 = {:.4f}'.format(np.mean(ap))) info_str += 'Mean AP@0.5 = {:.4f}\n\n'.format(np.mean(ap)) return info_str def do_python_eval_gen(self): """ python evaluation wrapper :return: info_str """ info_str = '' annopath = os.path.join(self.data_path, 'Annotations', '{0!s}.xml') imageset_file = os.path.join(self.data_path, 'ImageSets', self.image_set + '_eval.txt') annocache = os.path.join(self.cache_path, self.name + '_annotations.pkl') with open(imageset_file, 'w') as f: for i in range(len(self.pattern)): for j in range(self.frame_seg_len[i]): f.write((self.pattern[i] % (self.frame_seg_id[i] + j)) + ' ' + str(self.frame_id[i] + j) + '\n') filename = self.get_result_file_template().format('all') ap = vid_eval(filename, annopath, imageset_file, self.classes_map, annocache, ovthresh=0.5) for cls_ind, cls in enumerate(self.classes): if cls == '__background__': continue print('AP for {} = {:.4f}'.format(cls, ap[cls_ind-1])) info_str += 'AP for {} = {:.4f}\n'.format(cls, ap[cls_ind-1]) print('Mean AP@0.5 = {:.4f}'.format(np.mean(ap))) info_str += 'Mean AP@0.5 = {:.4f}\n\n'.format(np.mean(ap)) return info_str ================================================ FILE: lib/dataset/imagenet_vid_eval.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Xizhou Zhu # -------------------------------------------------------- """ given a imagenet vid imdb, compute mAP """ import numpy as np import os import cPickle def parse_vid_rec(filename, classhash, img_ids, defaultIOUthr=0.5, pixelTolerance=10): """ parse imagenet vid record into a dictionary :param filename: xml file path :return: list of dict """ import xml.etree.ElementTree as ET tree = ET.parse(filename) objects = [] for obj in tree.findall('object'): obj_dict = dict() obj_dict['label'] = classhash[obj.find('name').text] bbox = obj.find('bndbox') obj_dict['bbox'] = [float(bbox.find('xmin').text), float(bbox.find('ymin').text), float(bbox.find('xmax').text), float(bbox.find('ymax').text)] gt_w = obj_dict['bbox'][2] - obj_dict['bbox'][0] + 1 gt_h = obj_dict['bbox'][3] - obj_dict['bbox'][1] + 1 thr = (gt_w*gt_h)/((gt_w+pixelTolerance)*(gt_h+pixelTolerance)) obj_dict['thr'] = np.min([thr, defaultIOUthr]) objects.append(obj_dict) return {'bbox' : np.array([x['bbox'] for x in objects]), 'label': np.array([x['label'] for x in objects]), 'thr' : np.array([x['thr'] for x in objects]), 'img_ids': img_ids} def vid_ap(rec, prec): """ average precision calculations [precision integrated to recall] :param rec: recall :param prec: precision :return: average precision """ # append sentinel values at both ends mrec = np.concatenate(([0.], rec, [1.])) mpre = np.concatenate(([0.], prec, [0.])) # compute precision integration ladder for i in range(mpre.size - 1, 0, -1): mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i]) # look for recall value changes i = np.where(mrec[1:] != mrec[:-1])[0] # sum (\delta recall) * prec ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) return ap def vid_eval(detpath, annopath, imageset_file, classname_map, annocache, ovthresh=0.5): """ imagenet vid evaluation :param detpath: detection results detpath.format(classname) :param annopath: annotations annopath.format(classname) :param imageset_file: text file containing list of images :param annocache: caching annotations :param ovthresh: overlap threshold :return: rec, prec, ap """ with open(imageset_file, 'r') as f: lines = [x.strip().split(' ') for x in f.readlines()] img_basenames = [x[0] for x in lines] gt_img_ids = [int(x[1]) for x in lines] classhash = dict(zip(classname_map, range(0,len(classname_map)))) # load annotations from cache if not os.path.isfile(annocache): recs = [] for ind, image_filename in enumerate(img_basenames): recs.append(parse_vid_rec(annopath.format('VID/' + image_filename), classhash, gt_img_ids[ind])) if ind % 100 == 0: print 'reading annotations for {:d}/{:d}'.format(ind + 1, len(img_basenames)) print 'saving annotations cache to {:s}'.format(annocache) with open(annocache, 'wb') as f: cPickle.dump(recs, f, protocol=cPickle.HIGHEST_PROTOCOL) else: with open(annocache, 'rb') as f: recs = cPickle.load(f) # extract objects in :param classname: npos = np.zeros(len(classname_map)) for rec in recs: rec_labels = rec['label'] for x in rec_labels: npos[x] += 1 # read detections with open(detpath, 'r') as f: lines = f.readlines() splitlines = [x.strip().split(' ') for x in lines] img_ids = np.array([int(x[0]) for x in splitlines]) obj_labels = np.array([int(x[1]) for x in splitlines]) obj_confs = np.array([float(x[2]) for x in splitlines]) obj_bboxes = np.array([[float(z) for z in x[3:]] for x in splitlines]) # sort by confidence if obj_bboxes.shape[0] > 0: sorted_inds = np.argsort(img_ids) img_ids = img_ids[sorted_inds] obj_labels = obj_labels[sorted_inds] obj_confs = obj_confs[sorted_inds] obj_bboxes = obj_bboxes[sorted_inds, :] num_imgs = max(max(gt_img_ids),max(img_ids)) + 1 obj_labels_cell = [None] * num_imgs obj_confs_cell = [None] * num_imgs obj_bboxes_cell = [None] * num_imgs start_i = 0 id = img_ids[0] for i in range(0, len(img_ids)): if i == len(img_ids)-1 or img_ids[i+1] != id: conf = obj_confs[start_i:i+1] label = obj_labels[start_i:i+1] bbox = obj_bboxes[start_i:i+1, :] sorted_inds = np.argsort(-conf) obj_labels_cell[id] = label[sorted_inds] obj_confs_cell[id] = conf[sorted_inds] obj_bboxes_cell[id] = bbox[sorted_inds, :] if i < len(img_ids)-1: id = img_ids[i+1] start_i = i+1 # go down detections and mark true positives and false positives tp_cell = [None] * num_imgs fp_cell = [None] * num_imgs for rec in recs: id = rec['img_ids'] gt_labels = rec['label'] gt_bboxes = rec['bbox'] gt_thr = rec['thr'] num_gt_obj = len(gt_labels) gt_detected = np.zeros(num_gt_obj) labels = obj_labels_cell[id] bboxes = obj_bboxes_cell[id] num_obj = 0 if labels is None else len(labels) tp = np.zeros(num_obj) fp = np.zeros(num_obj) for j in range(0,num_obj): bb = bboxes[j, :] ovmax = -1 kmax = -1 for k in range(0,num_gt_obj): if labels[j] != gt_labels[k]: continue if gt_detected[k] > 0: continue bbgt = gt_bboxes[k, :] bi=[np.max((bb[0],bbgt[0])), np.max((bb[1],bbgt[1])), np.min((bb[2],bbgt[2])), np.min((bb[3],bbgt[3]))] iw=bi[2]-bi[0]+1 ih=bi[3]-bi[1]+1 if iw>0 and ih>0: # compute overlap as area of intersection / area of union ua = (bb[2] - bb[0] + 1.) * (bb[3] - bb[1] + 1.) + \ (bbgt[2] - bbgt[0] + 1.) * \ (bbgt[3] - bbgt[1] + 1.) - iw*ih ov=iw*ih/ua # makes sure that this object is detected according # to its individual threshold if ov >= gt_thr[k] and ov > ovmax: ovmax=ov kmax=k if kmax >= 0: tp[j] = 1 gt_detected[kmax] = 1 else: fp[j] = 1 tp_cell[id] = tp fp_cell[id] = fp tp_all = np.concatenate([x for x in np.array(tp_cell)[gt_img_ids] if x is not None]) fp_all = np.concatenate([x for x in np.array(fp_cell)[gt_img_ids] if x is not None]) obj_labels = np.concatenate([x for x in np.array(obj_labels_cell)[gt_img_ids] if x is not None]) confs = np.concatenate([x for x in np.array(obj_confs_cell)[gt_img_ids] if x is not None]) sorted_inds = np.argsort(-confs) tp_all = tp_all[sorted_inds] fp_all = fp_all[sorted_inds] obj_labels = obj_labels[sorted_inds] ap = np.zeros(len(classname_map)) for c in range(1, len(classname_map)): # compute precision recall fp = np.cumsum(fp_all[obj_labels == c]) tp = np.cumsum(tp_all[obj_labels == c]) rec = tp / float(npos[c]) # avoid division by zero in case first detection matches a difficult ground ruth prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps) ap[c] = vid_ap(rec, prec) ap = ap[1:] return ap ================================================ FILE: lib/dataset/imdb.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong # -------------------------------------------------------- """ General image database An image database creates a list of relative image path called image_set_index and transform index to absolute image path. As to training, it is necessary that ground truth and proposals are mixed together for training. roidb basic format [image_index] ['image', 'height', 'width', 'flipped', 'boxes', 'gt_classes', 'gt_overlaps', 'max_classes', 'max_overlaps', 'bbox_targets'] """ import os import cPickle import numpy as np from PIL import Image from bbox.bbox_transform import bbox_overlaps from multiprocessing import Pool, cpu_count def get_flipped_entry_outclass_wrapper(IMDB_instance, seg_rec): return IMDB_instance.get_flipped_entry(seg_rec) class IMDB(object): def __init__(self, name, image_set, root_path, dataset_path, result_path=None): """ basic information about an image database :param name: name of image database will be used for any output :param root_path: root path store cache and proposal data :param dataset_path: dataset path store images and image lists """ self.name = name + '_' + image_set self.image_set = image_set self.root_path = root_path self.data_path = dataset_path self._result_path = result_path # abstract attributes self.classes = [] self.num_classes = 0 self.image_set_index = [] self.num_images = 0 self.config = {} def image_path_from_index(self, index): raise NotImplementedError def gt_roidb(self): raise NotImplementedError def evaluate_detections(self, detections): raise NotImplementedError def evaluate_segmentations(self, segmentations): raise NotImplementedError @property def cache_path(self): """ make a directory to store all caches :return: cache path """ cache_path = os.path.join(self.root_path, 'cache') if not os.path.exists(cache_path): os.mkdir(cache_path) return cache_path @property def result_path(self): if self._result_path and os.path.exists(self._result_path): return self._result_path else: return self.cache_path def image_path_at(self, index): """ access image at index in image database :param index: image index in image database :return: image path """ return self.image_path_from_index(self.image_set_index[index]) def load_rpn_data(self, full=False): if full: rpn_file = os.path.join(self.result_path, 'rpn_data', self.name + '_full_rpn.pkl') else: rpn_file = os.path.join(self.result_path, 'rpn_data', self.name + '_rpn.pkl') print 'loading {}'.format(rpn_file) assert os.path.exists(rpn_file), 'rpn data not found at {}'.format(rpn_file) with open(rpn_file, 'rb') as f: box_list = cPickle.load(f) return box_list def load_rpn_roidb(self, gt_roidb): """ turn rpn detection boxes into roidb :param gt_roidb: [image_index]['boxes', 'gt_classes', 'gt_overlaps', 'flipped'] :return: roidb: [image_index]['boxes', 'gt_classes', 'gt_overlaps', 'flipped'] """ box_list = self.load_rpn_data() return self.create_roidb_from_box_list(box_list, gt_roidb) def rpn_roidb(self, gt_roidb, append_gt=False): """ get rpn roidb and ground truth roidb :param gt_roidb: ground truth roidb :param append_gt: append ground truth :return: roidb of rpn """ if append_gt: print 'appending ground truth annotations' rpn_roidb = self.load_rpn_roidb(gt_roidb) roidb = IMDB.merge_roidbs(gt_roidb, rpn_roidb) else: roidb = self.load_rpn_roidb(gt_roidb) return roidb def create_roidb_from_box_list(self, box_list, gt_roidb): """ given ground truth, prepare roidb :param box_list: [image_index] ndarray of [box_index][x1, x2, y1, y2] :param gt_roidb: [image_index]['boxes', 'gt_classes', 'gt_overlaps', 'flipped'] :return: roidb: [image_index]['boxes', 'gt_classes', 'gt_overlaps', 'flipped'] """ assert len(box_list) == self.num_images, 'number of boxes matrix must match number of images' roidb = [] for i in range(self.num_images): roi_rec = dict() roi_rec['image'] = gt_roidb[i]['image'] roi_rec['height'] = gt_roidb[i]['height'] roi_rec['width'] = gt_roidb[i]['width'] boxes = box_list[i] if boxes.shape[1] == 5: boxes = boxes[:, :4] num_boxes = boxes.shape[0] overlaps = np.zeros((num_boxes, self.num_classes), dtype=np.float32) if gt_roidb is not None and gt_roidb[i]['boxes'].size > 0: gt_boxes = gt_roidb[i]['boxes'] gt_classes = gt_roidb[i]['gt_classes'] # n boxes and k gt_boxes => n * k overlap gt_overlaps = bbox_overlaps(boxes.astype(np.float), gt_boxes.astype(np.float)) # for each box in n boxes, select only maximum overlap (must be greater than zero) argmaxes = gt_overlaps.argmax(axis=1) maxes = gt_overlaps.max(axis=1) I = np.where(maxes > 0)[0] overlaps[I, gt_classes[argmaxes[I]]] = maxes[I] roi_rec.update({'boxes': boxes, 'gt_classes': np.zeros((num_boxes,), dtype=np.int32), 'gt_overlaps': overlaps, 'max_classes': overlaps.argmax(axis=1), 'max_overlaps': overlaps.max(axis=1), 'flipped': False}) # background roi => background class zero_indexes = np.where(roi_rec['max_overlaps'] == 0)[0] assert all(roi_rec['max_classes'][zero_indexes] == 0) # foreground roi => foreground class nonzero_indexes = np.where(roi_rec['max_overlaps'] > 0)[0] assert all(roi_rec['max_classes'][nonzero_indexes] != 0) roidb.append(roi_rec) return roidb def get_flipped_entry(self, seg_rec): return {'image': self.flip_and_save(seg_rec['image']), 'seg_cls_path': self.flip_and_save(seg_rec['seg_cls_path']), 'height': seg_rec['height'], 'width': seg_rec['width'], 'flipped': True} def append_flipped_images_for_segmentation(self, segdb): """ append flipped images to an roidb flip boxes coordinates, images will be actually flipped when loading into network :param segdb: [image_index]['seg_cls_path', 'flipped'] :return: segdb: [image_index]['seg_cls_path', 'flipped'] """ print 'append flipped images to segdb' assert self.num_images == len(segdb) pool = Pool(processes=1) pool_result = [] for i in range(self.num_images): seg_rec = segdb[i] pool_result.append(pool.apply_async(get_flipped_entry_outclass_wrapper, args=(self, seg_rec, ))) #self.get_flipped_entry(seg_rec, segdb_flip, i) pool.close() pool.join() segdb_flip = [res_instance.get() for res_instance in pool_result] segdb += segdb_flip self.image_set_index *= 2 return segdb def append_flipped_images(self, roidb): """ append flipped images to an roidb flip boxes coordinates, images will be actually flipped when loading into network :param roidb: [image_index]['boxes', 'gt_classes', 'gt_overlaps', 'flipped'] :return: roidb: [image_index]['boxes', 'gt_classes', 'gt_overlaps', 'flipped'] """ print 'append flipped images to roidb' assert self.num_images == len(roidb) for i in range(self.num_images): roi_rec = roidb[i] boxes = roi_rec['boxes'].copy() oldx1 = boxes[:, 0].copy() oldx2 = boxes[:, 2].copy() boxes[:, 0] = roi_rec['width'] - oldx2 - 1 boxes[:, 2] = roi_rec['width'] - oldx1 - 1 assert (boxes[:, 2] >= boxes[:, 0]).all() entry = roi_rec.copy() entry['boxes'] = boxes entry['flipped'] = True # if roidb has mask if 'cache_seg_inst' in roi_rec: [filename, extension] = os.path.splitext(roi_rec['cache_seg_inst']) entry['cache_seg_inst'] = os.path.join(filename + '_flip' + extension) roidb.append(entry) self.image_set_index *= 2 return roidb def flip_and_save(self, image_path): """ flip the image by the path and save the flipped image with suffix 'flip' :param path: the path of specific image :return: the path of saved image """ [image_name, image_ext] = os.path.splitext(os.path.basename(image_path)) image_dir = os.path.dirname(image_path) saved_image_path = os.path.join(image_dir, image_name + '_flip' + image_ext) try: flipped_image = Image.open(saved_image_path) except: flipped_image = Image.open(image_path) flipped_image = flipped_image.transpose(Image.FLIP_LEFT_RIGHT) flipped_image.save(saved_image_path, 'png') return saved_image_path def evaluate_recall(self, roidb, candidate_boxes=None, thresholds=None): """ evaluate detection proposal recall metrics record max overlap value for each gt box; return vector of overlap values :param roidb: used to evaluate :param candidate_boxes: if not given, use roidb's non-gt boxes :param thresholds: array-like recall threshold :return: None ar: average recall, recalls: vector recalls at each IoU overlap threshold thresholds: vector of IoU overlap threshold, gt_overlaps: vector of all ground-truth overlaps """ all_log_info = '' area_names = ['all', '0-25', '25-50', '50-100', '100-200', '200-300', '300-inf'] area_ranges = [[0**2, 1e5**2], [0**2, 25**2], [25**2, 50**2], [50**2, 100**2], [100**2, 200**2], [200**2, 300**2], [300**2, 1e5**2]] area_counts = [] for area_name, area_range in zip(area_names[1:], area_ranges[1:]): area_count = 0 for i in range(self.num_images): if candidate_boxes is None: # default is use the non-gt boxes from roidb non_gt_inds = np.where(roidb[i]['gt_classes'] == 0)[0] boxes = roidb[i]['boxes'][non_gt_inds, :] else: boxes = candidate_boxes[i] boxes_areas = (boxes[:, 2] - boxes[:, 0] + 1) * (boxes[:, 3] - boxes[:, 1] + 1) valid_range_inds = np.where((boxes_areas >= area_range[0]) & (boxes_areas < area_range[1]))[0] area_count += len(valid_range_inds) area_counts.append(area_count) total_counts = float(sum(area_counts)) for area_name, area_count in zip(area_names[1:], area_counts): log_info = 'percentage of {} {}'.format(area_name, area_count / total_counts) print log_info all_log_info += log_info log_info = 'average number of proposal {}'.format(total_counts / self.num_images) print log_info all_log_info += log_info for area_name, area_range in zip(area_names, area_ranges): gt_overlaps = np.zeros(0) num_pos = 0 for i in range(self.num_images): # check for max_overlaps == 1 avoids including crowd annotations max_gt_overlaps = roidb[i]['gt_overlaps'].max(axis=1) gt_inds = np.where((roidb[i]['gt_classes'] > 0) & (max_gt_overlaps == 1))[0] gt_boxes = roidb[i]['boxes'][gt_inds, :] gt_areas = (gt_boxes[:, 2] - gt_boxes[:, 0] + 1) * (gt_boxes[:, 3] - gt_boxes[:, 1] + 1) valid_gt_inds = np.where((gt_areas >= area_range[0]) & (gt_areas < area_range[1]))[0] gt_boxes = gt_boxes[valid_gt_inds, :] num_pos += len(valid_gt_inds) if candidate_boxes is None: # default is use the non-gt boxes from roidb non_gt_inds = np.where(roidb[i]['gt_classes'] == 0)[0] boxes = roidb[i]['boxes'][non_gt_inds, :] else: boxes = candidate_boxes[i] if boxes.shape[0] == 0: continue overlaps = bbox_overlaps(boxes.astype(np.float), gt_boxes.astype(np.float)) _gt_overlaps = np.zeros((gt_boxes.shape[0])) # choose whatever is smaller to iterate rounds = min(boxes.shape[0], gt_boxes.shape[0]) for j in range(rounds): # find which proposal maximally covers each gt box argmax_overlaps = overlaps.argmax(axis=0) # get the IoU amount of coverage for each gt box max_overlaps = overlaps.max(axis=0) # find which gt box is covered by most IoU gt_ind = max_overlaps.argmax() gt_ovr = max_overlaps.max() assert (gt_ovr >= 0), '%s\n%s\n%s' % (boxes, gt_boxes, overlaps) # find the proposal box that covers the best covered gt box box_ind = argmax_overlaps[gt_ind] # record the IoU coverage of this gt box _gt_overlaps[j] = overlaps[box_ind, gt_ind] assert (_gt_overlaps[j] == gt_ovr) # mark the proposal box and the gt box as used overlaps[box_ind, :] = -1 overlaps[:, gt_ind] = -1 # append recorded IoU coverage level gt_overlaps = np.hstack((gt_overlaps, _gt_overlaps)) gt_overlaps = np.sort(gt_overlaps) if thresholds is None: step = 0.05 thresholds = np.arange(0.5, 0.95 + 1e-5, step) recalls = np.zeros_like(thresholds) # compute recall for each IoU threshold for i, t in enumerate(thresholds): recalls[i] = (gt_overlaps >= t).sum() / float(num_pos) ar = recalls.mean() # print results log_info = 'average recall for {}: {:.3f}'.format(area_name, ar) print log_info all_log_info += log_info for threshold, recall in zip(thresholds, recalls): log_info = 'recall @{:.2f}: {:.3f}'.format(threshold, recall) print log_info all_log_info += log_info return all_log_info @staticmethod def merge_roidbs(a, b): """ merge roidbs into one :param a: roidb to be merged into :param b: roidb to be merged :return: merged imdb """ assert len(a) == len(b) for i in range(len(a)): a[i]['boxes'] = np.vstack((a[i]['boxes'], b[i]['boxes'])) a[i]['gt_classes'] = np.hstack((a[i]['gt_classes'], b[i]['gt_classes'])) a[i]['gt_overlaps'] = np.vstack((a[i]['gt_overlaps'], b[i]['gt_overlaps'])) a[i]['max_classes'] = np.hstack((a[i]['max_classes'], b[i]['max_classes'])) a[i]['max_overlaps'] = np.hstack((a[i]['max_overlaps'], b[i]['max_overlaps'])) return a ================================================ FILE: lib/nms/__init__.py ================================================ ================================================ FILE: lib/nms/cpu_nms.pyx ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong # -------------------------------------------------------- # Based on: # py-faster-rcnn # Copyright (c) 2015 Microsoft # Licence under The MIT License # https://github.com/rbgirshick/py-faster-rcnn # -------------------------------------------------------- import numpy as np cimport numpy as np cdef inline np.float32_t max(np.float32_t a, np.float32_t b): return a if a >= b else b cdef inline np.float32_t min(np.float32_t a, np.float32_t b): return a if a <= b else b def cpu_nms(np.ndarray[np.float32_t, ndim=2] dets, np.float thresh): cdef np.ndarray[np.float32_t, ndim=1] x1 = dets[:, 0] cdef np.ndarray[np.float32_t, ndim=1] y1 = dets[:, 1] cdef np.ndarray[np.float32_t, ndim=1] x2 = dets[:, 2] cdef np.ndarray[np.float32_t, ndim=1] y2 = dets[:, 3] cdef np.ndarray[np.float32_t, ndim=1] scores = dets[:, 4] cdef np.ndarray[np.float32_t, ndim=1] areas = (x2 - x1 + 1) * (y2 - y1 + 1) cdef np.ndarray[np.int_t, ndim=1] order = scores.argsort()[::-1].astype('i') cdef int ndets = dets.shape[0] cdef np.ndarray[np.int_t, ndim=1] suppressed = \ np.zeros((ndets), dtype=np.int) # nominal indices cdef int _i, _j # sorted indices cdef int i, j # temp variables for box i's (the box currently under consideration) cdef np.float32_t ix1, iy1, ix2, iy2, iarea # variables for computing overlap with box j (lower scoring box) cdef np.float32_t xx1, yy1, xx2, yy2 cdef np.float32_t w, h cdef np.float32_t inter, ovr keep = [] for _i in range(ndets): i = order[_i] if suppressed[i] == 1: continue keep.append(i) ix1 = x1[i] iy1 = y1[i] ix2 = x2[i] iy2 = y2[i] iarea = areas[i] for _j in range(_i + 1, ndets): j = order[_j] if suppressed[j] == 1: continue xx1 = max(ix1, x1[j]) yy1 = max(iy1, y1[j]) xx2 = min(ix2, x2[j]) yy2 = min(iy2, y2[j]) w = max(0.0, xx2 - xx1 + 1) h = max(0.0, yy2 - yy1 + 1) inter = w * h ovr = inter / (iarea + areas[j] - inter) if ovr >= thresh: suppressed[j] = 1 return keep ================================================ FILE: lib/nms/gpu_nms.cu ================================================ // ------------------------------------------------------------------ // Deep Feature Flow // Copyright (c) 2017 Microsoft // Licensed under The MIT License // Written by Yuwen Xiong // ------------------------------------------------------------------ // Based on: // Faster R-CNN // Copyright (c) 2015 Microsoft // Licensed under The MIT License // https://github.com/shaoqingren/faster_rcnn // ------------------------------------------------------------------ //#include "gpu_nms.hpp" #include #include #define CUDA_CHECK(condition) \ /* Code block avoids redefinition of cudaError_t error */ \ do { \ cudaError_t error = condition; \ if (error != cudaSuccess) { \ std::cout << cudaGetErrorString(error) << std::endl; \ } \ } while (0) #define DIVUP(m,n) ((m) / (n) + ((m) % (n) > 0)) int const threadsPerBlock = sizeof(unsigned long long) * 8; __device__ inline float devIoU(float const * const a, float const * const b) { float left = max(a[0], b[0]), right = min(a[2], b[2]); float top = max(a[1], b[1]), bottom = min(a[3], b[3]); float width = max(right - left + 1, 0.f), height = max(bottom - top + 1, 0.f); float interS = width * height; float Sa = (a[2] - a[0] + 1) * (a[3] - a[1] + 1); float Sb = (b[2] - b[0] + 1) * (b[3] - b[1] + 1); return interS / (Sa + Sb - interS); } __global__ void nms_kernel(const int n_boxes, const float nms_overlap_thresh, const float *dev_boxes, unsigned long long *dev_mask) { const int row_start = blockIdx.y; const int col_start = blockIdx.x; // if (row_start > col_start) return; const int row_size = min(n_boxes - row_start * threadsPerBlock, threadsPerBlock); const int col_size = min(n_boxes - col_start * threadsPerBlock, threadsPerBlock); __shared__ float block_boxes[threadsPerBlock * 5]; if (threadIdx.x < col_size) { block_boxes[threadIdx.x * 5 + 0] = dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 0]; block_boxes[threadIdx.x * 5 + 1] = dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 1]; block_boxes[threadIdx.x * 5 + 2] = dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 2]; block_boxes[threadIdx.x * 5 + 3] = dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 3]; block_boxes[threadIdx.x * 5 + 4] = dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 4]; } __syncthreads(); if (threadIdx.x < row_size) { const int cur_box_idx = threadsPerBlock * row_start + threadIdx.x; const float *cur_box = dev_boxes + cur_box_idx * 5; int i = 0; unsigned long long t = 0; int start = 0; if (row_start == col_start) { start = threadIdx.x + 1; } for (i = start; i < col_size; i++) { if (devIoU(cur_box, block_boxes + i * 5) > nms_overlap_thresh) { t |= 1ULL << i; } } const int col_blocks = DIVUP(n_boxes, threadsPerBlock); dev_mask[cur_box_idx * col_blocks + col_start] = t; } } void _set_device(int device_id) { int current_device; CUDA_CHECK(cudaGetDevice(¤t_device)); if (current_device == device_id) { return; } // The call to cudaSetDevice must come before any calls to Get, which // may perform initialization using the GPU. CUDA_CHECK(cudaSetDevice(device_id)); } void _nms(long* keep_out, int* num_out, const float* boxes_host, int boxes_num, int boxes_dim, float nms_overlap_thresh, int device_id) { _set_device(device_id); float* boxes_dev = NULL; unsigned long long* mask_dev = NULL; const int col_blocks = DIVUP(boxes_num, threadsPerBlock); CUDA_CHECK(cudaMalloc(&boxes_dev, boxes_num * boxes_dim * sizeof(float))); CUDA_CHECK(cudaMemcpy(boxes_dev, boxes_host, boxes_num * boxes_dim * sizeof(float), cudaMemcpyHostToDevice)); CUDA_CHECK(cudaMalloc(&mask_dev, boxes_num * col_blocks * sizeof(unsigned long long))); dim3 blocks(DIVUP(boxes_num, threadsPerBlock), DIVUP(boxes_num, threadsPerBlock)); dim3 threads(threadsPerBlock); nms_kernel<<>>(boxes_num, nms_overlap_thresh, boxes_dev, mask_dev); std::vector mask_host(boxes_num * col_blocks); CUDA_CHECK(cudaMemcpy(&mask_host[0], mask_dev, sizeof(unsigned long long) * boxes_num * col_blocks, cudaMemcpyDeviceToHost)); std::vector remv(col_blocks); memset(&remv[0], 0, sizeof(unsigned long long) * col_blocks); int num_to_keep = 0; for (int i = 0; i < boxes_num; i++) { int nblock = i / threadsPerBlock; int inblock = i % threadsPerBlock; if (!(remv[nblock] & (1ULL << inblock))) { keep_out[num_to_keep++] = i; unsigned long long *p = &mask_host[0] + i * col_blocks; for (int j = nblock; j < col_blocks; j++) { remv[j] |= p[j]; } } } *num_out = num_to_keep; CUDA_CHECK(cudaFree(boxes_dev)); CUDA_CHECK(cudaFree(mask_dev)); } /* Generated by Cython 0.24 */ #define PY_SSIZE_T_CLEAN #include "Python.h" #ifndef Py_PYTHON_H #error Python headers needed to compile C extensions, please install development version of Python. #elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03020000) #error Cython requires Python 2.6+ or Python 3.2+. #else #define CYTHON_ABI "0_24" #include #ifndef offsetof #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) #endif #if !defined(WIN32) && !defined(MS_WINDOWS) #ifndef __stdcall #define __stdcall #endif #ifndef __cdecl #define __cdecl #endif #ifndef __fastcall #define __fastcall #endif #endif #ifndef DL_IMPORT #define DL_IMPORT(t) t #endif #ifndef DL_EXPORT #define DL_EXPORT(t) t #endif #ifndef PY_LONG_LONG #define PY_LONG_LONG LONG_LONG #endif #ifndef Py_HUGE_VAL #define Py_HUGE_VAL HUGE_VAL #endif #ifdef PYPY_VERSION #define CYTHON_COMPILING_IN_PYPY 1 #define CYTHON_COMPILING_IN_CPYTHON 0 #else #define CYTHON_COMPILING_IN_PYPY 0 #define CYTHON_COMPILING_IN_CPYTHON 1 #endif #if !defined(CYTHON_USE_PYLONG_INTERNALS) && CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x02070000 #define CYTHON_USE_PYLONG_INTERNALS 1 #endif #if CYTHON_USE_PYLONG_INTERNALS #include "longintrepr.h" #undef SHIFT #undef BASE #undef MASK #endif #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x02070600 && !defined(Py_OptimizeFlag) #define Py_OptimizeFlag 0 #endif #define __PYX_BUILD_PY_SSIZE_T "n" #define CYTHON_FORMAT_SSIZE_T "z" #if PY_MAJOR_VERSION < 3 #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) #define __Pyx_DefaultClassType PyClass_Type #else #define __Pyx_BUILTIN_MODULE_NAME "builtins" #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) #define __Pyx_DefaultClassType PyType_Type #endif #ifndef Py_TPFLAGS_CHECKTYPES #define Py_TPFLAGS_CHECKTYPES 0 #endif #ifndef Py_TPFLAGS_HAVE_INDEX #define Py_TPFLAGS_HAVE_INDEX 0 #endif #ifndef Py_TPFLAGS_HAVE_NEWBUFFER #define Py_TPFLAGS_HAVE_NEWBUFFER 0 #endif #ifndef Py_TPFLAGS_HAVE_FINALIZE #define Py_TPFLAGS_HAVE_FINALIZE 0 #endif #if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) #define CYTHON_PEP393_ENABLED 1 #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ 0 : _PyUnicode_Ready((PyObject *)(op))) #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) #define __Pyx_PyUnicode_KIND(u) PyUnicode_KIND(u) #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) #else #define CYTHON_PEP393_ENABLED 0 #define __Pyx_PyUnicode_READY(op) (0) #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u) #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i])) #define __Pyx_PyUnicode_KIND(u) (sizeof(Py_UNICODE)) #define __Pyx_PyUnicode_DATA(u) ((void*)PyUnicode_AS_UNICODE(u)) #define __Pyx_PyUnicode_READ(k, d, i) ((void)(k), (Py_UCS4)(((Py_UNICODE*)d)[i])) #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_SIZE(u)) #endif #if CYTHON_COMPILING_IN_PYPY #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) #else #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) #endif #if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_Contains) #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) #endif #if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Format) #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) #endif #if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) #define PyObject_Malloc(s) PyMem_Malloc(s) #define PyObject_Free(p) PyMem_Free(p) #define PyObject_Realloc(p) PyMem_Realloc(p) #endif #define __Pyx_PyString_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : __Pyx_PyString_Format(a, b)) #define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) #if PY_MAJOR_VERSION >= 3 #define __Pyx_PyString_Format(a, b) PyUnicode_Format(a, b) #else #define __Pyx_PyString_Format(a, b) PyString_Format(a, b) #endif #if PY_MAJOR_VERSION < 3 && !defined(PyObject_ASCII) #define PyObject_ASCII(o) PyObject_Repr(o) #endif #if PY_MAJOR_VERSION >= 3 #define PyBaseString_Type PyUnicode_Type #define PyStringObject PyUnicodeObject #define PyString_Type PyUnicode_Type #define PyString_Check PyUnicode_Check #define PyString_CheckExact PyUnicode_CheckExact #endif #if PY_MAJOR_VERSION >= 3 #define __Pyx_PyBaseString_Check(obj) PyUnicode_Check(obj) #define __Pyx_PyBaseString_CheckExact(obj) PyUnicode_CheckExact(obj) #else #define __Pyx_PyBaseString_Check(obj) (PyString_Check(obj) || PyUnicode_Check(obj)) #define __Pyx_PyBaseString_CheckExact(obj) (PyString_CheckExact(obj) || PyUnicode_CheckExact(obj)) #endif #ifndef PySet_CheckExact #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) #endif #define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) #if PY_MAJOR_VERSION >= 3 #define PyIntObject PyLongObject #define PyInt_Type PyLong_Type #define PyInt_Check(op) PyLong_Check(op) #define PyInt_CheckExact(op) PyLong_CheckExact(op) #define PyInt_FromString PyLong_FromString #define PyInt_FromUnicode PyLong_FromUnicode #define PyInt_FromLong PyLong_FromLong #define PyInt_FromSize_t PyLong_FromSize_t #define PyInt_FromSsize_t PyLong_FromSsize_t #define PyInt_AsLong PyLong_AsLong #define PyInt_AS_LONG PyLong_AS_LONG #define PyInt_AsSsize_t PyLong_AsSsize_t #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask #define PyNumber_Int PyNumber_Long #endif #if PY_MAJOR_VERSION >= 3 #define PyBoolObject PyLongObject #endif #if PY_MAJOR_VERSION >= 3 && CYTHON_COMPILING_IN_PYPY #ifndef PyUnicode_InternFromString #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) #endif #endif #if PY_VERSION_HEX < 0x030200A4 typedef long Py_hash_t; #define __Pyx_PyInt_FromHash_t PyInt_FromLong #define __Pyx_PyInt_AsHash_t PyInt_AsLong #else #define __Pyx_PyInt_FromHash_t PyInt_FromSsize_t #define __Pyx_PyInt_AsHash_t PyInt_AsSsize_t #endif #if PY_MAJOR_VERSION >= 3 #define __Pyx_PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : PyInstanceMethod_New(func)) #else #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass) #endif #if PY_VERSION_HEX >= 0x030500B1 #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods #define __Pyx_PyType_AsAsync(obj) (Py_TYPE(obj)->tp_as_async) #elif CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 typedef struct { unaryfunc am_await; unaryfunc am_aiter; unaryfunc am_anext; } __Pyx_PyAsyncMethodsStruct; #define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved)) #else #define __Pyx_PyType_AsAsync(obj) NULL #endif #ifndef CYTHON_RESTRICT #if defined(__GNUC__) #define CYTHON_RESTRICT __restrict__ #elif defined(_MSC_VER) && _MSC_VER >= 1400 #define CYTHON_RESTRICT __restrict #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L #define CYTHON_RESTRICT restrict #else #define CYTHON_RESTRICT #endif #endif #define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) #ifndef __cplusplus #error "Cython files generated with the C++ option must be compiled with a C++ compiler." #endif #ifndef CYTHON_INLINE #define CYTHON_INLINE inline #endif template void __Pyx_call_destructor(T& x) { x.~T(); } template class __Pyx_FakeReference { public: __Pyx_FakeReference() : ptr(NULL) { } __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } T *operator->() { return ptr; } operator T&() { return *ptr; } private: T *ptr; }; #if defined(WIN32) || defined(MS_WINDOWS) #define _USE_MATH_DEFINES #endif #include #ifdef NAN #define __PYX_NAN() ((float) NAN) #else static CYTHON_INLINE float __PYX_NAN() { float value; memset(&value, 0xFF, sizeof(value)); return value; } #endif #define __PYX_ERR(f_index, lineno, Ln_error) \ { \ __pyx_filename = __pyx_f[f_index]; __pyx_lineno = lineno; __pyx_clineno = __LINE__; goto Ln_error; \ } #if PY_MAJOR_VERSION >= 3 #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) #else #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) #endif #ifndef __PYX_EXTERN_C #ifdef __cplusplus #define __PYX_EXTERN_C extern "C" #else #define __PYX_EXTERN_C extern #endif #endif #define __PYX_HAVE__nms__gpu_nms #define __PYX_HAVE_API__nms__gpu_nms #include "string.h" #include "stdio.h" #include "stdlib.h" #include "numpy/arrayobject.h" #include "numpy/ufuncobject.h" #include "gpu_nms.hpp" #ifdef _OPENMP #include #endif /* _OPENMP */ #ifdef PYREX_WITHOUT_ASSERTIONS #define CYTHON_WITHOUT_ASSERTIONS #endif #ifndef CYTHON_UNUSED # if defined(__GNUC__) # if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) # define CYTHON_UNUSED __attribute__ ((__unused__)) # else # define CYTHON_UNUSED # endif # elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) # define CYTHON_UNUSED __attribute__ ((__unused__)) # else # define CYTHON_UNUSED # endif #endif #ifndef CYTHON_NCP_UNUSED # if CYTHON_COMPILING_IN_CPYTHON # define CYTHON_NCP_UNUSED # else # define CYTHON_NCP_UNUSED CYTHON_UNUSED # endif #endif typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* encoding; const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; #define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 #define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT 0 #define __PYX_DEFAULT_STRING_ENCODING "" #define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString #define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize #define __Pyx_uchar_cast(c) ((unsigned char)c) #define __Pyx_long_cast(x) ((long)x) #define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ (sizeof(type) < sizeof(Py_ssize_t)) ||\ (sizeof(type) > sizeof(Py_ssize_t) &&\ likely(v < (type)PY_SSIZE_T_MAX ||\ v == (type)PY_SSIZE_T_MAX) &&\ (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ v == (type)PY_SSIZE_T_MIN))) ||\ (sizeof(type) == sizeof(Py_ssize_t) &&\ (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ v == (type)PY_SSIZE_T_MAX))) ) #if defined (__cplusplus) && __cplusplus >= 201103L #include #define __Pyx_sst_abs(value) std::abs(value) #elif SIZEOF_INT >= SIZEOF_SIZE_T #define __Pyx_sst_abs(value) abs(value) #elif SIZEOF_LONG >= SIZEOF_SIZE_T #define __Pyx_sst_abs(value) labs(value) #elif defined (_MSC_VER) && defined (_M_X64) #define __Pyx_sst_abs(value) _abs64(value) #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L #define __Pyx_sst_abs(value) llabs(value) #elif defined (__GNUC__) #define __Pyx_sst_abs(value) __builtin_llabs(value) #else #define __Pyx_sst_abs(value) ((value<0) ? -value : value) #endif static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject*); static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); #define __Pyx_PyByteArray_FromString(s) PyByteArray_FromStringAndSize((const char*)s, strlen((const char*)s)) #define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) #define __Pyx_PyBytes_FromString PyBytes_FromString #define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); #if PY_MAJOR_VERSION < 3 #define __Pyx_PyStr_FromString __Pyx_PyBytes_FromString #define __Pyx_PyStr_FromStringAndSize __Pyx_PyBytes_FromStringAndSize #else #define __Pyx_PyStr_FromString __Pyx_PyUnicode_FromString #define __Pyx_PyStr_FromStringAndSize __Pyx_PyUnicode_FromStringAndSize #endif #define __Pyx_PyObject_AsSString(s) ((signed char*) __Pyx_PyObject_AsString(s)) #define __Pyx_PyObject_AsUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) #define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) #define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) #define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) #define __Pyx_PyStr_FromCString(s) __Pyx_PyStr_FromString((const char*)s) #define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) #if PY_MAJOR_VERSION < 3 static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) { const Py_UNICODE *u_end = u; while (*u_end++) ; return (size_t)(u_end - u - 1); } #else #define __Pyx_Py_UNICODE_strlen Py_UNICODE_strlen #endif #define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u)) #define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode #define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode #define __Pyx_NewRef(obj) (Py_INCREF(obj), obj) #define __Pyx_Owned_Py_None(b) __Pyx_NewRef(Py_None) #define __Pyx_PyBool_FromLong(b) ((b) ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False)) static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x); static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); #if CYTHON_COMPILING_IN_CPYTHON #define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) #else #define __pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) #endif #define __pyx_PyFloat_AsFloat(x) ((float) __pyx_PyFloat_AsDouble(x)) #if PY_MAJOR_VERSION >= 3 #define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) #else #define __Pyx_PyNumber_Int(x) (PyInt_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Int(x)) #endif #define __Pyx_PyNumber_Float(x) (PyFloat_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Float(x)) #if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII static int __Pyx_sys_getdefaultencoding_not_ascii; static int __Pyx_init_sys_getdefaultencoding_params(void) { PyObject* sys; PyObject* default_encoding = NULL; PyObject* ascii_chars_u = NULL; PyObject* ascii_chars_b = NULL; const char* default_encoding_c; sys = PyImport_ImportModule("sys"); if (!sys) goto bad; default_encoding = PyObject_CallMethod(sys, (char*) "getdefaultencoding", NULL); Py_DECREF(sys); if (!default_encoding) goto bad; default_encoding_c = PyBytes_AsString(default_encoding); if (!default_encoding_c) goto bad; if (strcmp(default_encoding_c, "ascii") == 0) { __Pyx_sys_getdefaultencoding_not_ascii = 0; } else { char ascii_chars[128]; int c; for (c = 0; c < 128; c++) { ascii_chars[c] = c; } __Pyx_sys_getdefaultencoding_not_ascii = 1; ascii_chars_u = PyUnicode_DecodeASCII(ascii_chars, 128, NULL); if (!ascii_chars_u) goto bad; ascii_chars_b = PyUnicode_AsEncodedString(ascii_chars_u, default_encoding_c, NULL); if (!ascii_chars_b || !PyBytes_Check(ascii_chars_b) || memcmp(ascii_chars, PyBytes_AS_STRING(ascii_chars_b), 128) != 0) { PyErr_Format( PyExc_ValueError, "This module compiled with c_string_encoding=ascii, but default encoding '%.200s' is not a superset of ascii.", default_encoding_c); goto bad; } Py_DECREF(ascii_chars_u); Py_DECREF(ascii_chars_b); } Py_DECREF(default_encoding); return 0; bad: Py_XDECREF(default_encoding); Py_XDECREF(ascii_chars_u); Py_XDECREF(ascii_chars_b); return -1; } #endif #if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT && PY_MAJOR_VERSION >= 3 #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) #else #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) #if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT static char* __PYX_DEFAULT_STRING_ENCODING; static int __Pyx_init_sys_getdefaultencoding_params(void) { PyObject* sys; PyObject* default_encoding = NULL; char* default_encoding_c; sys = PyImport_ImportModule("sys"); if (!sys) goto bad; default_encoding = PyObject_CallMethod(sys, (char*) (const char*) "getdefaultencoding", NULL); Py_DECREF(sys); if (!default_encoding) goto bad; default_encoding_c = PyBytes_AsString(default_encoding); if (!default_encoding_c) goto bad; __PYX_DEFAULT_STRING_ENCODING = (char*) malloc(strlen(default_encoding_c)); if (!__PYX_DEFAULT_STRING_ENCODING) goto bad; strcpy(__PYX_DEFAULT_STRING_ENCODING, default_encoding_c); Py_DECREF(default_encoding); return 0; bad: Py_XDECREF(default_encoding); return -1; } #endif #endif /* Test for GCC > 2.95 */ #if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) #define likely(x) __builtin_expect(!!(x), 1) #define unlikely(x) __builtin_expect(!!(x), 0) #else /* !__GNUC__ or GCC < 2.95 */ #define likely(x) (x) #define unlikely(x) (x) #endif /* __GNUC__ */ static PyObject *__pyx_m; static PyObject *__pyx_d; static PyObject *__pyx_b; static PyObject *__pyx_empty_tuple; static PyObject *__pyx_empty_bytes; static PyObject *__pyx_empty_unicode; static int __pyx_lineno; static int __pyx_clineno = 0; static const char * __pyx_cfilenm= __FILE__; static const char *__pyx_filename; /* None.proto */ #if !defined(CYTHON_CCOMPLEX) #if defined(__cplusplus) #define CYTHON_CCOMPLEX 1 #elif defined(_Complex_I) #define CYTHON_CCOMPLEX 1 #else #define CYTHON_CCOMPLEX 0 #endif #endif #if CYTHON_CCOMPLEX #ifdef __cplusplus #include #else #include #endif #endif #if CYTHON_CCOMPLEX && !defined(__cplusplus) && defined(__sun__) && defined(__GNUC__) #undef _Complex_I #define _Complex_I 1.0fj #endif static const char *__pyx_f[] = { "nms\\gpu_nms.pyx", "__init__.pxd", "type.pxd", }; /* BufferFormatStructs.proto */ #define IS_UNSIGNED(type) (((type) -1) > 0) struct __Pyx_StructField_; #define __PYX_BUF_FLAGS_PACKED_STRUCT (1 << 0) typedef struct { const char* name; struct __Pyx_StructField_* fields; size_t size; size_t arraysize[8]; int ndim; char typegroup; char is_unsigned; int flags; } __Pyx_TypeInfo; typedef struct __Pyx_StructField_ { __Pyx_TypeInfo* type; const char* name; size_t offset; } __Pyx_StructField; typedef struct { __Pyx_StructField* field; size_t parent_offset; } __Pyx_BufFmt_StackElem; typedef struct { __Pyx_StructField root; __Pyx_BufFmt_StackElem* head; size_t fmt_offset; size_t new_count, enc_count; size_t struct_alignment; int is_complex; char enc_type; char new_packmode; char enc_packmode; char is_valid_array; } __Pyx_BufFmt_Context; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":725 * # in Cython to enable them only on the right systems. * * ctypedef npy_int8 int8_t # <<<<<<<<<<<<<< * ctypedef npy_int16 int16_t * ctypedef npy_int32 int32_t */ typedef npy_int8 __pyx_t_5numpy_int8_t; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":726 * * ctypedef npy_int8 int8_t * ctypedef npy_int16 int16_t # <<<<<<<<<<<<<< * ctypedef npy_int32 int32_t * ctypedef npy_int64 int64_t */ typedef npy_int16 __pyx_t_5numpy_int16_t; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":727 * ctypedef npy_int8 int8_t * ctypedef npy_int16 int16_t * ctypedef npy_int32 int32_t # <<<<<<<<<<<<<< * ctypedef npy_int64 int64_t * #ctypedef npy_int96 int96_t */ typedef npy_int32 __pyx_t_5numpy_int32_t; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":728 * ctypedef npy_int16 int16_t * ctypedef npy_int32 int32_t * ctypedef npy_int64 int64_t # <<<<<<<<<<<<<< * #ctypedef npy_int96 int96_t * #ctypedef npy_int128 int128_t */ typedef npy_int64 __pyx_t_5numpy_int64_t; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":732 * #ctypedef npy_int128 int128_t * * ctypedef npy_uint8 uint8_t # <<<<<<<<<<<<<< * ctypedef npy_uint16 uint16_t * ctypedef npy_uint32 uint32_t */ typedef npy_uint8 __pyx_t_5numpy_uint8_t; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":733 * * ctypedef npy_uint8 uint8_t * ctypedef npy_uint16 uint16_t # <<<<<<<<<<<<<< * ctypedef npy_uint32 uint32_t * ctypedef npy_uint64 uint64_t */ typedef npy_uint16 __pyx_t_5numpy_uint16_t; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":734 * ctypedef npy_uint8 uint8_t * ctypedef npy_uint16 uint16_t * ctypedef npy_uint32 uint32_t # <<<<<<<<<<<<<< * ctypedef npy_uint64 uint64_t * #ctypedef npy_uint96 uint96_t */ typedef npy_uint32 __pyx_t_5numpy_uint32_t; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":735 * ctypedef npy_uint16 uint16_t * ctypedef npy_uint32 uint32_t * ctypedef npy_uint64 uint64_t # <<<<<<<<<<<<<< * #ctypedef npy_uint96 uint96_t * #ctypedef npy_uint128 uint128_t */ typedef npy_uint64 __pyx_t_5numpy_uint64_t; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":739 * #ctypedef npy_uint128 uint128_t * * ctypedef npy_float32 float32_t # <<<<<<<<<<<<<< * ctypedef npy_float64 float64_t * #ctypedef npy_float80 float80_t */ typedef npy_float32 __pyx_t_5numpy_float32_t; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":740 * * ctypedef npy_float32 float32_t * ctypedef npy_float64 float64_t # <<<<<<<<<<<<<< * #ctypedef npy_float80 float80_t * #ctypedef npy_float128 float128_t */ typedef npy_float64 __pyx_t_5numpy_float64_t; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":749 * # The int types are mapped a bit surprising -- * # numpy.int corresponds to 'l' and numpy.long to 'q' * ctypedef npy_long int_t # <<<<<<<<<<<<<< * ctypedef npy_longlong long_t * ctypedef npy_longlong longlong_t */ typedef npy_long __pyx_t_5numpy_int_t; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":750 * # numpy.int corresponds to 'l' and numpy.long to 'q' * ctypedef npy_long int_t * ctypedef npy_longlong long_t # <<<<<<<<<<<<<< * ctypedef npy_longlong longlong_t * */ typedef npy_longlong __pyx_t_5numpy_long_t; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":751 * ctypedef npy_long int_t * ctypedef npy_longlong long_t * ctypedef npy_longlong longlong_t # <<<<<<<<<<<<<< * * ctypedef npy_ulong uint_t */ typedef npy_longlong __pyx_t_5numpy_longlong_t; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":753 * ctypedef npy_longlong longlong_t * * ctypedef npy_ulong uint_t # <<<<<<<<<<<<<< * ctypedef npy_ulonglong ulong_t * ctypedef npy_ulonglong ulonglong_t */ typedef npy_ulong __pyx_t_5numpy_uint_t; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":754 * * ctypedef npy_ulong uint_t * ctypedef npy_ulonglong ulong_t # <<<<<<<<<<<<<< * ctypedef npy_ulonglong ulonglong_t * */ typedef npy_ulonglong __pyx_t_5numpy_ulong_t; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":755 * ctypedef npy_ulong uint_t * ctypedef npy_ulonglong ulong_t * ctypedef npy_ulonglong ulonglong_t # <<<<<<<<<<<<<< * * ctypedef npy_intp intp_t */ typedef npy_ulonglong __pyx_t_5numpy_ulonglong_t; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":757 * ctypedef npy_ulonglong ulonglong_t * * ctypedef npy_intp intp_t # <<<<<<<<<<<<<< * ctypedef npy_uintp uintp_t * */ typedef npy_intp __pyx_t_5numpy_intp_t; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":758 * * ctypedef npy_intp intp_t * ctypedef npy_uintp uintp_t # <<<<<<<<<<<<<< * * ctypedef npy_double float_t */ typedef npy_uintp __pyx_t_5numpy_uintp_t; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":760 * ctypedef npy_uintp uintp_t * * ctypedef npy_double float_t # <<<<<<<<<<<<<< * ctypedef npy_double double_t * ctypedef npy_longdouble longdouble_t */ typedef npy_double __pyx_t_5numpy_float_t; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":761 * * ctypedef npy_double float_t * ctypedef npy_double double_t # <<<<<<<<<<<<<< * ctypedef npy_longdouble longdouble_t * */ typedef npy_double __pyx_t_5numpy_double_t; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":762 * ctypedef npy_double float_t * ctypedef npy_double double_t * ctypedef npy_longdouble longdouble_t # <<<<<<<<<<<<<< * * ctypedef npy_cfloat cfloat_t */ typedef npy_longdouble __pyx_t_5numpy_longdouble_t; /* None.proto */ #if CYTHON_CCOMPLEX #ifdef __cplusplus typedef ::std::complex< float > __pyx_t_float_complex; #else typedef float _Complex __pyx_t_float_complex; #endif #else typedef struct { float real, imag; } __pyx_t_float_complex; #endif /* None.proto */ #if CYTHON_CCOMPLEX #ifdef __cplusplus typedef ::std::complex< double > __pyx_t_double_complex; #else typedef double _Complex __pyx_t_double_complex; #endif #else typedef struct { double real, imag; } __pyx_t_double_complex; #endif /*--- Type declarations ---*/ /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":764 * ctypedef npy_longdouble longdouble_t * * ctypedef npy_cfloat cfloat_t # <<<<<<<<<<<<<< * ctypedef npy_cdouble cdouble_t * ctypedef npy_clongdouble clongdouble_t */ typedef npy_cfloat __pyx_t_5numpy_cfloat_t; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":765 * * ctypedef npy_cfloat cfloat_t * ctypedef npy_cdouble cdouble_t # <<<<<<<<<<<<<< * ctypedef npy_clongdouble clongdouble_t * */ typedef npy_cdouble __pyx_t_5numpy_cdouble_t; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":766 * ctypedef npy_cfloat cfloat_t * ctypedef npy_cdouble cdouble_t * ctypedef npy_clongdouble clongdouble_t # <<<<<<<<<<<<<< * * ctypedef npy_cdouble complex_t */ typedef npy_clongdouble __pyx_t_5numpy_clongdouble_t; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":768 * ctypedef npy_clongdouble clongdouble_t * * ctypedef npy_cdouble complex_t # <<<<<<<<<<<<<< * * cdef inline object PyArray_MultiIterNew1(a): */ typedef npy_cdouble __pyx_t_5numpy_complex_t; /* --- Runtime support code (head) --- */ /* Refnanny.proto */ #ifndef CYTHON_REFNANNY #define CYTHON_REFNANNY 0 #endif #if CYTHON_REFNANNY typedef struct { void (*INCREF)(void*, PyObject*, int); void (*DECREF)(void*, PyObject*, int); void (*GOTREF)(void*, PyObject*, int); void (*GIVEREF)(void*, PyObject*, int); void* (*SetupContext)(const char*, int, const char*); void (*FinishContext)(void**); } __Pyx_RefNannyAPIStruct; static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; #ifdef WITH_THREAD #define __Pyx_RefNannySetupContext(name, acquire_gil)\ if (acquire_gil) {\ PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ PyGILState_Release(__pyx_gilstate_save);\ } else {\ __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ } #else #define __Pyx_RefNannySetupContext(name, acquire_gil)\ __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__) #endif #define __Pyx_RefNannyFinishContext()\ __Pyx_RefNanny->FinishContext(&__pyx_refnanny) #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__) #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__) #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__) #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__) #define __Pyx_XINCREF(r) do { if((r) != NULL) {__Pyx_INCREF(r); }} while(0) #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r); }} while(0) #define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r); }} while(0) #define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);}} while(0) #else #define __Pyx_RefNannyDeclarations #define __Pyx_RefNannySetupContext(name, acquire_gil) #define __Pyx_RefNannyFinishContext() #define __Pyx_INCREF(r) Py_INCREF(r) #define __Pyx_DECREF(r) Py_DECREF(r) #define __Pyx_GOTREF(r) #define __Pyx_GIVEREF(r) #define __Pyx_XINCREF(r) Py_XINCREF(r) #define __Pyx_XDECREF(r) Py_XDECREF(r) #define __Pyx_XGOTREF(r) #define __Pyx_XGIVEREF(r) #endif #define __Pyx_XDECREF_SET(r, v) do {\ PyObject *tmp = (PyObject *) r;\ r = v; __Pyx_XDECREF(tmp);\ } while (0) #define __Pyx_DECREF_SET(r, v) do {\ PyObject *tmp = (PyObject *) r;\ r = v; __Pyx_DECREF(tmp);\ } while (0) #define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) #define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) /* RaiseArgTupleInvalid.proto */ static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); /* RaiseDoubleKeywords.proto */ static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); /* ParseKeywords.proto */ static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[],\ PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args,\ const char* function_name); /* ArgTypeTest.proto */ static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, const char *name, int exact); /* BufferFormatCheck.proto */ static CYTHON_INLINE int __Pyx_GetBufferAndValidate(Py_buffer* buf, PyObject* obj, __Pyx_TypeInfo* dtype, int flags, int nd, int cast, __Pyx_BufFmt_StackElem* stack); static CYTHON_INLINE void __Pyx_SafeReleaseBuffer(Py_buffer* info); static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const char* ts); static void __Pyx_BufFmt_Init(__Pyx_BufFmt_Context* ctx, __Pyx_BufFmt_StackElem* stack, __Pyx_TypeInfo* type); // PROTO /* PyObjectGetAttrStr.proto */ #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { PyTypeObject* tp = Py_TYPE(obj); if (likely(tp->tp_getattro)) return tp->tp_getattro(obj, attr_name); #if PY_MAJOR_VERSION < 3 if (likely(tp->tp_getattr)) return tp->tp_getattr(obj, PyString_AS_STRING(attr_name)); #endif return PyObject_GetAttr(obj, attr_name); } #else #define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) #endif /* GetBuiltinName.proto */ static PyObject *__Pyx_GetBuiltinName(PyObject *name); /* GetModuleGlobalName.proto */ static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name); /* PyObjectCall.proto */ #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); #else #define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) #endif /* ExtTypeTest.proto */ static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type); /* PyObjectCallMethO.proto */ #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); #endif /* PyObjectCallOneArg.proto */ static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); /* PyObjectCallNoArg.proto */ #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func); #else #define __Pyx_PyObject_CallNoArg(func) __Pyx_PyObject_Call(func, __pyx_empty_tuple, NULL) #endif /* BufferIndexError.proto */ static void __Pyx_RaiseBufferIndexError(int axis); #define __Pyx_BufPtrStrided1d(type, buf, i0, s0) (type)((char*)buf + i0 * s0) #define __Pyx_BufPtrStrided2d(type, buf, i0, s0, i1, s1) (type)((char*)buf + i0 * s0 + i1 * s1) /* SliceObject.proto */ static CYTHON_INLINE PyObject* __Pyx_PyObject_GetSlice( PyObject* obj, Py_ssize_t cstart, Py_ssize_t cstop, PyObject** py_start, PyObject** py_stop, PyObject** py_slice, int has_cstart, int has_cstop, int wraparound); /* BufferFallbackError.proto */ static void __Pyx_RaiseBufferFallbackError(void); /* PyThreadStateGet.proto */ #if CYTHON_COMPILING_IN_CPYTHON #define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; #define __Pyx_PyThreadState_assign __pyx_tstate = PyThreadState_GET(); #else #define __Pyx_PyThreadState_declare #define __Pyx_PyThreadState_assign #endif /* PyErrFetchRestore.proto */ #if CYTHON_COMPILING_IN_CPYTHON #define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) #define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) #define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) #define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); #else #define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) #define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) #define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) #define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) #endif /* RaiseException.proto */ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); /* DictGetItem.proto */ #if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY static PyObject *__Pyx_PyDict_GetItem(PyObject *d, PyObject* key) { PyObject *value; value = PyDict_GetItemWithError(d, key); if (unlikely(!value)) { if (!PyErr_Occurred()) { PyObject* args = PyTuple_Pack(1, key); if (likely(args)) PyErr_SetObject(PyExc_KeyError, args); Py_XDECREF(args); } return NULL; } Py_INCREF(value); return value; } #else #define __Pyx_PyDict_GetItem(d, key) PyObject_GetItem(d, key) #endif /* RaiseTooManyValuesToUnpack.proto */ static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected); /* RaiseNeedMoreValuesToUnpack.proto */ static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index); /* RaiseNoneIterError.proto */ static CYTHON_INLINE void __Pyx_RaiseNoneNotIterableError(void); /* Import.proto */ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); /* CodeObjectCache.proto */ typedef struct { PyCodeObject* code_object; int code_line; } __Pyx_CodeObjectCacheEntry; struct __Pyx_CodeObjectCache { int count; int max_count; __Pyx_CodeObjectCacheEntry* entries; }; static struct __Pyx_CodeObjectCache __pyx_code_cache = {0,0,NULL}; static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); static PyCodeObject *__pyx_find_code_object(int code_line); static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object); /* AddTraceback.proto */ static void __Pyx_AddTraceback(const char *funcname, int c_line, int py_line, const char *filename); /* BufferStructDeclare.proto */ typedef struct { Py_ssize_t shape, strides, suboffsets; } __Pyx_Buf_DimInfo; typedef struct { size_t refcount; Py_buffer pybuffer; } __Pyx_Buffer; typedef struct { __Pyx_Buffer *rcbuffer; char *data; __Pyx_Buf_DimInfo diminfo[8]; } __Pyx_LocalBuf_ND; #if PY_MAJOR_VERSION < 3 static int __Pyx_GetBuffer(PyObject *obj, Py_buffer *view, int flags); static void __Pyx_ReleaseBuffer(Py_buffer *view); #else #define __Pyx_GetBuffer PyObject_GetBuffer #define __Pyx_ReleaseBuffer PyBuffer_Release #endif /* None.proto */ static Py_ssize_t __Pyx_zeros[] = {0, 0, 0, 0, 0, 0, 0, 0}; static Py_ssize_t __Pyx_minusones[] = {-1, -1, -1, -1, -1, -1, -1, -1}; /* CIntToPy.proto */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value); /* None.proto */ #if CYTHON_CCOMPLEX #ifdef __cplusplus #define __Pyx_CREAL(z) ((z).real()) #define __Pyx_CIMAG(z) ((z).imag()) #else #define __Pyx_CREAL(z) (__real__(z)) #define __Pyx_CIMAG(z) (__imag__(z)) #endif #else #define __Pyx_CREAL(z) ((z).real) #define __Pyx_CIMAG(z) ((z).imag) #endif #if defined(__cplusplus) && CYTHON_CCOMPLEX && (defined(_WIN32) || defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 5 || __GNUC__ == 4 && __GNUC_MINOR__ >= 4 )) || __cplusplus >= 201103) #define __Pyx_SET_CREAL(z,x) ((z).real(x)) #define __Pyx_SET_CIMAG(z,y) ((z).imag(y)) #else #define __Pyx_SET_CREAL(z,x) __Pyx_CREAL(z) = (x) #define __Pyx_SET_CIMAG(z,y) __Pyx_CIMAG(z) = (y) #endif /* None.proto */ static CYTHON_INLINE __pyx_t_float_complex __pyx_t_float_complex_from_parts(float, float); /* None.proto */ #if CYTHON_CCOMPLEX #define __Pyx_c_eqf(a, b) ((a)==(b)) #define __Pyx_c_sumf(a, b) ((a)+(b)) #define __Pyx_c_difff(a, b) ((a)-(b)) #define __Pyx_c_prodf(a, b) ((a)*(b)) #define __Pyx_c_quotf(a, b) ((a)/(b)) #define __Pyx_c_negf(a) (-(a)) #ifdef __cplusplus #define __Pyx_c_is_zerof(z) ((z)==(float)0) #define __Pyx_c_conjf(z) (::std::conj(z)) #if 1 #define __Pyx_c_absf(z) (::std::abs(z)) #define __Pyx_c_powf(a, b) (::std::pow(a, b)) #endif #else #define __Pyx_c_is_zerof(z) ((z)==0) #define __Pyx_c_conjf(z) (conjf(z)) #if 1 #define __Pyx_c_absf(z) (cabsf(z)) #define __Pyx_c_powf(a, b) (cpowf(a, b)) #endif #endif #else static CYTHON_INLINE int __Pyx_c_eqf(__pyx_t_float_complex, __pyx_t_float_complex); static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_sumf(__pyx_t_float_complex, __pyx_t_float_complex); static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_difff(__pyx_t_float_complex, __pyx_t_float_complex); static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_prodf(__pyx_t_float_complex, __pyx_t_float_complex); static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_quotf(__pyx_t_float_complex, __pyx_t_float_complex); static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_negf(__pyx_t_float_complex); static CYTHON_INLINE int __Pyx_c_is_zerof(__pyx_t_float_complex); static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_conjf(__pyx_t_float_complex); #if 1 static CYTHON_INLINE float __Pyx_c_absf(__pyx_t_float_complex); static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_powf(__pyx_t_float_complex, __pyx_t_float_complex); #endif #endif /* None.proto */ static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(double, double); /* None.proto */ #if CYTHON_CCOMPLEX #define __Pyx_c_eq(a, b) ((a)==(b)) #define __Pyx_c_sum(a, b) ((a)+(b)) #define __Pyx_c_diff(a, b) ((a)-(b)) #define __Pyx_c_prod(a, b) ((a)*(b)) #define __Pyx_c_quot(a, b) ((a)/(b)) #define __Pyx_c_neg(a) (-(a)) #ifdef __cplusplus #define __Pyx_c_is_zero(z) ((z)==(double)0) #define __Pyx_c_conj(z) (::std::conj(z)) #if 1 #define __Pyx_c_abs(z) (::std::abs(z)) #define __Pyx_c_pow(a, b) (::std::pow(a, b)) #endif #else #define __Pyx_c_is_zero(z) ((z)==0) #define __Pyx_c_conj(z) (conj(z)) #if 1 #define __Pyx_c_abs(z) (cabs(z)) #define __Pyx_c_pow(a, b) (cpow(a, b)) #endif #endif #else static CYTHON_INLINE int __Pyx_c_eq(__pyx_t_double_complex, __pyx_t_double_complex); static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_sum(__pyx_t_double_complex, __pyx_t_double_complex); static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_diff(__pyx_t_double_complex, __pyx_t_double_complex); static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_prod(__pyx_t_double_complex, __pyx_t_double_complex); static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_quot(__pyx_t_double_complex, __pyx_t_double_complex); static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_neg(__pyx_t_double_complex); static CYTHON_INLINE int __Pyx_c_is_zero(__pyx_t_double_complex); static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_conj(__pyx_t_double_complex); #if 1 static CYTHON_INLINE double __Pyx_c_abs(__pyx_t_double_complex); static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_pow(__pyx_t_double_complex, __pyx_t_double_complex); #endif #endif /* CIntToPy.proto */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_enum__NPY_TYPES(enum NPY_TYPES value); /* CIntFromPy.proto */ static CYTHON_INLINE npy_int32 __Pyx_PyInt_As_npy_int32(PyObject *); /* CIntFromPy.proto */ static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); /* CIntToPy.proto */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); /* CIntFromPy.proto */ static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); /* CheckBinaryVersion.proto */ static int __Pyx_check_binary_version(void); /* PyIdentifierFromString.proto */ #if !defined(__Pyx_PyIdentifier_FromString) #if PY_MAJOR_VERSION < 3 #define __Pyx_PyIdentifier_FromString(s) PyString_FromString(s) #else #define __Pyx_PyIdentifier_FromString(s) PyUnicode_FromString(s) #endif #endif /* ModuleImport.proto */ static PyObject *__Pyx_ImportModule(const char *name); /* TypeImport.proto */ static PyTypeObject *__Pyx_ImportType(const char *module_name, const char *class_name, size_t size, int strict); /* InitStrings.proto */ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /* Module declarations from 'cpython.buffer' */ /* Module declarations from 'libc.string' */ /* Module declarations from 'libc.stdio' */ /* Module declarations from '__builtin__' */ /* Module declarations from 'cpython.type' */ static PyTypeObject *__pyx_ptype_7cpython_4type_type = 0; /* Module declarations from 'cpython' */ /* Module declarations from 'cpython.object' */ /* Module declarations from 'cpython.ref' */ /* Module declarations from 'libc.stdlib' */ /* Module declarations from 'numpy' */ /* Module declarations from 'numpy' */ static PyTypeObject *__pyx_ptype_5numpy_dtype = 0; static PyTypeObject *__pyx_ptype_5numpy_flatiter = 0; static PyTypeObject *__pyx_ptype_5numpy_broadcast = 0; static PyTypeObject *__pyx_ptype_5numpy_ndarray = 0; static PyTypeObject *__pyx_ptype_5numpy_ufunc = 0; static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *, char *, char *, int *); /*proto*/ /* Module declarations from 'nms.gpu_nms' */ static __Pyx_TypeInfo __Pyx_TypeInfo_nn___pyx_t_5numpy_float32_t = { "float32_t", NULL, sizeof(__pyx_t_5numpy_float32_t), { 0 }, 0, 'R', 0, 0 }; static __Pyx_TypeInfo __Pyx_TypeInfo_nn___pyx_t_5numpy_int32_t = { "int32_t", NULL, sizeof(__pyx_t_5numpy_int32_t), { 0 }, 0, IS_UNSIGNED(__pyx_t_5numpy_int32_t) ? 'U' : 'I', IS_UNSIGNED(__pyx_t_5numpy_int32_t), 0 }; static __Pyx_TypeInfo __Pyx_TypeInfo_nn___pyx_t_5numpy_intp_t = { "intp_t", NULL, sizeof(__pyx_t_5numpy_intp_t), { 0 }, 0, IS_UNSIGNED(__pyx_t_5numpy_intp_t) ? 'U' : 'I', IS_UNSIGNED(__pyx_t_5numpy_intp_t), 0 }; #define __Pyx_MODULE_NAME "nms.gpu_nms" int __pyx_module_is_main_nms__gpu_nms = 0; /* Implementation of 'nms.gpu_nms' */ static PyObject *__pyx_builtin_ValueError; static PyObject *__pyx_builtin_range; static PyObject *__pyx_builtin_RuntimeError; static const char __pyx_k_np[] = "np"; static const char __pyx_k_dets[] = "dets"; static const char __pyx_k_keep[] = "keep"; static const char __pyx_k_main[] = "__main__"; static const char __pyx_k_test[] = "__test__"; static const char __pyx_k_dtype[] = "dtype"; static const char __pyx_k_int32[] = "int32"; static const char __pyx_k_numpy[] = "numpy"; static const char __pyx_k_order[] = "order"; static const char __pyx_k_range[] = "range"; static const char __pyx_k_zeros[] = "zeros"; static const char __pyx_k_import[] = "__import__"; static const char __pyx_k_scores[] = "scores"; static const char __pyx_k_thresh[] = "thresh"; static const char __pyx_k_argsort[] = "argsort"; static const char __pyx_k_gpu_nms[] = "gpu_nms"; static const char __pyx_k_num_out[] = "num_out"; static const char __pyx_k_boxes_dim[] = "boxes_dim"; static const char __pyx_k_boxes_num[] = "boxes_num"; static const char __pyx_k_device_id[] = "device_id"; static const char __pyx_k_ValueError[] = "ValueError"; static const char __pyx_k_nms_gpu_nms[] = "nms.gpu_nms"; static const char __pyx_k_sorted_dets[] = "sorted_dets"; static const char __pyx_k_RuntimeError[] = "RuntimeError"; static const char __pyx_k_ndarray_is_not_C_contiguous[] = "ndarray is not C contiguous"; static const char __pyx_k_unknown_dtype_code_in_numpy_pxd[] = "unknown dtype code in numpy.pxd (%d)"; static const char __pyx_k_D_v_zix_caffe_caffe_win_20160523[] = "D:\\v-zix\\caffe\\caffe-win-20160523\\models\\py-faster-rcnn-windows\\lib\\nms\\gpu_nms.pyx"; static const char __pyx_k_Format_string_allocated_too_shor[] = "Format string allocated too short, see comment in numpy.pxd"; static const char __pyx_k_Non_native_byte_order_not_suppor[] = "Non-native byte order not supported"; static const char __pyx_k_ndarray_is_not_Fortran_contiguou[] = "ndarray is not Fortran contiguous"; static const char __pyx_k_Format_string_allocated_too_shor_2[] = "Format string allocated too short."; static PyObject *__pyx_kp_s_D_v_zix_caffe_caffe_win_20160523; static PyObject *__pyx_kp_u_Format_string_allocated_too_shor; static PyObject *__pyx_kp_u_Format_string_allocated_too_shor_2; static PyObject *__pyx_kp_u_Non_native_byte_order_not_suppor; static PyObject *__pyx_n_s_RuntimeError; static PyObject *__pyx_n_s_ValueError; static PyObject *__pyx_n_s_argsort; static PyObject *__pyx_n_s_boxes_dim; static PyObject *__pyx_n_s_boxes_num; static PyObject *__pyx_n_s_dets; static PyObject *__pyx_n_s_device_id; static PyObject *__pyx_n_s_dtype; static PyObject *__pyx_n_s_gpu_nms; static PyObject *__pyx_n_s_import; static PyObject *__pyx_n_s_int32; static PyObject *__pyx_n_s_keep; static PyObject *__pyx_n_s_main; static PyObject *__pyx_kp_u_ndarray_is_not_C_contiguous; static PyObject *__pyx_kp_u_ndarray_is_not_Fortran_contiguou; static PyObject *__pyx_n_s_nms_gpu_nms; static PyObject *__pyx_n_s_np; static PyObject *__pyx_n_s_num_out; static PyObject *__pyx_n_s_numpy; static PyObject *__pyx_n_s_order; static PyObject *__pyx_n_s_range; static PyObject *__pyx_n_s_scores; static PyObject *__pyx_n_s_sorted_dets; static PyObject *__pyx_n_s_test; static PyObject *__pyx_n_s_thresh; static PyObject *__pyx_kp_u_unknown_dtype_code_in_numpy_pxd; static PyObject *__pyx_n_s_zeros; static PyObject *__pyx_pf_3nms_7gpu_nms_gpu_nms(CYTHON_UNUSED PyObject *__pyx_self, PyArrayObject *__pyx_v_dets, PyObject *__pyx_v_thresh, __pyx_t_5numpy_int32_t __pyx_v_device_id); /* proto */ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /* proto */ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info); /* proto */ static PyObject *__pyx_int_4; static PyObject *__pyx_int_neg_1; static PyObject *__pyx_slice_; static PyObject *__pyx_slice__3; static PyObject *__pyx_slice__4; static PyObject *__pyx_tuple__2; static PyObject *__pyx_tuple__5; static PyObject *__pyx_tuple__6; static PyObject *__pyx_tuple__7; static PyObject *__pyx_tuple__8; static PyObject *__pyx_tuple__9; static PyObject *__pyx_tuple__10; static PyObject *__pyx_tuple__11; static PyObject *__pyx_codeobj__12; /* "nms/gpu_nms.pyx":16 * void _nms(np.int32_t*, int*, np.float32_t*, int, int, float, int) * * def gpu_nms(np.ndarray[np.float32_t, ndim=2] dets, np.float thresh, # <<<<<<<<<<<<<< * np.int32_t device_id=0): * cdef int boxes_num = dets.shape[0] */ /* Python wrapper */ static PyObject *__pyx_pw_3nms_7gpu_nms_1gpu_nms(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static PyMethodDef __pyx_mdef_3nms_7gpu_nms_1gpu_nms = {"gpu_nms", (PyCFunction)__pyx_pw_3nms_7gpu_nms_1gpu_nms, METH_VARARGS|METH_KEYWORDS, 0}; static PyObject *__pyx_pw_3nms_7gpu_nms_1gpu_nms(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyArrayObject *__pyx_v_dets = 0; PyObject *__pyx_v_thresh = 0; __pyx_t_5numpy_int32_t __pyx_v_device_id; PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("gpu_nms (wrapper)", 0); { static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_dets,&__pyx_n_s_thresh,&__pyx_n_s_device_id,0}; PyObject* values[3] = {0,0,0}; if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); case 0: break; default: goto __pyx_L5_argtuple_error; } kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_dets)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_thresh)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("gpu_nms", 0, 2, 3, 1); __PYX_ERR(0, 16, __pyx_L3_error) } case 2: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_device_id); if (value) { values[2] = value; kw_args--; } } } if (unlikely(kw_args > 0)) { if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "gpu_nms") < 0)) __PYX_ERR(0, 16, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); values[0] = PyTuple_GET_ITEM(__pyx_args, 0); break; default: goto __pyx_L5_argtuple_error; } } __pyx_v_dets = ((PyArrayObject *)values[0]); __pyx_v_thresh = ((PyObject*)values[1]); if (values[2]) { __pyx_v_device_id = __Pyx_PyInt_As_npy_int32(values[2]); if (unlikely((__pyx_v_device_id == (npy_int32)-1) && PyErr_Occurred())) __PYX_ERR(0, 17, __pyx_L3_error) } else { __pyx_v_device_id = ((__pyx_t_5numpy_int32_t)0); } } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; __Pyx_RaiseArgtupleInvalid("gpu_nms", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 16, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("nms.gpu_nms.gpu_nms", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_dets), __pyx_ptype_5numpy_ndarray, 1, "dets", 0))) __PYX_ERR(0, 16, __pyx_L1_error) if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_thresh), (&PyFloat_Type), 1, "thresh", 1))) __PYX_ERR(0, 16, __pyx_L1_error) __pyx_r = __pyx_pf_3nms_7gpu_nms_gpu_nms(__pyx_self, __pyx_v_dets, __pyx_v_thresh, __pyx_v_device_id); /* function exit code */ goto __pyx_L0; __pyx_L1_error:; __pyx_r = NULL; __pyx_L0:; __Pyx_RefNannyFinishContext(); return __pyx_r; } static PyObject *__pyx_pf_3nms_7gpu_nms_gpu_nms(CYTHON_UNUSED PyObject *__pyx_self, PyArrayObject *__pyx_v_dets, PyObject *__pyx_v_thresh, __pyx_t_5numpy_int32_t __pyx_v_device_id) { int __pyx_v_boxes_num; int __pyx_v_boxes_dim; int __pyx_v_num_out; PyArrayObject *__pyx_v_keep = 0; PyArrayObject *__pyx_v_scores = 0; PyArrayObject *__pyx_v_order = 0; PyArrayObject *__pyx_v_sorted_dets = 0; __Pyx_LocalBuf_ND __pyx_pybuffernd_dets; __Pyx_Buffer __pyx_pybuffer_dets; __Pyx_LocalBuf_ND __pyx_pybuffernd_keep; __Pyx_Buffer __pyx_pybuffer_keep; __Pyx_LocalBuf_ND __pyx_pybuffernd_order; __Pyx_Buffer __pyx_pybuffer_order; __Pyx_LocalBuf_ND __pyx_pybuffernd_scores; __Pyx_Buffer __pyx_pybuffer_scores; __Pyx_LocalBuf_ND __pyx_pybuffernd_sorted_dets; __Pyx_Buffer __pyx_pybuffer_sorted_dets; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; PyObject *__pyx_t_4 = NULL; PyObject *__pyx_t_5 = NULL; PyArrayObject *__pyx_t_6 = NULL; PyArrayObject *__pyx_t_7 = NULL; PyArrayObject *__pyx_t_8 = NULL; PyArrayObject *__pyx_t_9 = NULL; Py_ssize_t __pyx_t_10; int __pyx_t_11; Py_ssize_t __pyx_t_12; Py_ssize_t __pyx_t_13; float __pyx_t_14; PyObject *__pyx_t_15 = NULL; PyObject *__pyx_t_16 = NULL; PyObject *__pyx_t_17 = NULL; __Pyx_RefNannySetupContext("gpu_nms", 0); __pyx_pybuffer_keep.pybuffer.buf = NULL; __pyx_pybuffer_keep.refcount = 0; __pyx_pybuffernd_keep.data = NULL; __pyx_pybuffernd_keep.rcbuffer = &__pyx_pybuffer_keep; __pyx_pybuffer_scores.pybuffer.buf = NULL; __pyx_pybuffer_scores.refcount = 0; __pyx_pybuffernd_scores.data = NULL; __pyx_pybuffernd_scores.rcbuffer = &__pyx_pybuffer_scores; __pyx_pybuffer_order.pybuffer.buf = NULL; __pyx_pybuffer_order.refcount = 0; __pyx_pybuffernd_order.data = NULL; __pyx_pybuffernd_order.rcbuffer = &__pyx_pybuffer_order; __pyx_pybuffer_sorted_dets.pybuffer.buf = NULL; __pyx_pybuffer_sorted_dets.refcount = 0; __pyx_pybuffernd_sorted_dets.data = NULL; __pyx_pybuffernd_sorted_dets.rcbuffer = &__pyx_pybuffer_sorted_dets; __pyx_pybuffer_dets.pybuffer.buf = NULL; __pyx_pybuffer_dets.refcount = 0; __pyx_pybuffernd_dets.data = NULL; __pyx_pybuffernd_dets.rcbuffer = &__pyx_pybuffer_dets; { __Pyx_BufFmt_StackElem __pyx_stack[1]; if (unlikely(__Pyx_GetBufferAndValidate(&__pyx_pybuffernd_dets.rcbuffer->pybuffer, (PyObject*)__pyx_v_dets, &__Pyx_TypeInfo_nn___pyx_t_5numpy_float32_t, PyBUF_FORMAT| PyBUF_STRIDES, 2, 0, __pyx_stack) == -1)) __PYX_ERR(0, 16, __pyx_L1_error) } __pyx_pybuffernd_dets.diminfo[0].strides = __pyx_pybuffernd_dets.rcbuffer->pybuffer.strides[0]; __pyx_pybuffernd_dets.diminfo[0].shape = __pyx_pybuffernd_dets.rcbuffer->pybuffer.shape[0]; __pyx_pybuffernd_dets.diminfo[1].strides = __pyx_pybuffernd_dets.rcbuffer->pybuffer.strides[1]; __pyx_pybuffernd_dets.diminfo[1].shape = __pyx_pybuffernd_dets.rcbuffer->pybuffer.shape[1]; /* "nms/gpu_nms.pyx":18 * def gpu_nms(np.ndarray[np.float32_t, ndim=2] dets, np.float thresh, * np.int32_t device_id=0): * cdef int boxes_num = dets.shape[0] # <<<<<<<<<<<<<< * cdef int boxes_dim = dets.shape[1] * cdef int num_out */ __pyx_v_boxes_num = (__pyx_v_dets->dimensions[0]); /* "nms/gpu_nms.pyx":19 * np.int32_t device_id=0): * cdef int boxes_num = dets.shape[0] * cdef int boxes_dim = dets.shape[1] # <<<<<<<<<<<<<< * cdef int num_out * cdef np.ndarray[np.int32_t, ndim=1] \ */ __pyx_v_boxes_dim = (__pyx_v_dets->dimensions[1]); /* "nms/gpu_nms.pyx":22 * cdef int num_out * cdef np.ndarray[np.int32_t, ndim=1] \ * keep = np.zeros(boxes_num, dtype=np.int32) # <<<<<<<<<<<<<< * cdef np.ndarray[np.float32_t, ndim=1] \ * scores = dets[:, 4] */ __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 22, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_zeros); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 22, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_boxes_num); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 22, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 22, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1); __pyx_t_1 = 0; __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 22, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_np); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 22, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_int32); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 22, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_dtype, __pyx_t_5) < 0) __PYX_ERR(0, 22, __pyx_L1_error) __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_3, __pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 22, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (!(likely(((__pyx_t_5) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_5, __pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 22, __pyx_L1_error) __pyx_t_6 = ((PyArrayObject *)__pyx_t_5); { __Pyx_BufFmt_StackElem __pyx_stack[1]; if (unlikely(__Pyx_GetBufferAndValidate(&__pyx_pybuffernd_keep.rcbuffer->pybuffer, (PyObject*)__pyx_t_6, &__Pyx_TypeInfo_nn___pyx_t_5numpy_int32_t, PyBUF_FORMAT| PyBUF_STRIDES, 1, 0, __pyx_stack) == -1)) { __pyx_v_keep = ((PyArrayObject *)Py_None); __Pyx_INCREF(Py_None); __pyx_pybuffernd_keep.rcbuffer->pybuffer.buf = NULL; __PYX_ERR(0, 21, __pyx_L1_error) } else {__pyx_pybuffernd_keep.diminfo[0].strides = __pyx_pybuffernd_keep.rcbuffer->pybuffer.strides[0]; __pyx_pybuffernd_keep.diminfo[0].shape = __pyx_pybuffernd_keep.rcbuffer->pybuffer.shape[0]; } } __pyx_t_6 = 0; __pyx_v_keep = ((PyArrayObject *)__pyx_t_5); __pyx_t_5 = 0; /* "nms/gpu_nms.pyx":24 * keep = np.zeros(boxes_num, dtype=np.int32) * cdef np.ndarray[np.float32_t, ndim=1] \ * scores = dets[:, 4] # <<<<<<<<<<<<<< * #cdef np.ndarray[np.int_t, ndim=1] \ // 20160601, by xzn * # order = scores.argsort()[::-1] */ __pyx_t_5 = PyObject_GetItem(((PyObject *)__pyx_v_dets), __pyx_tuple__2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 24, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); if (!(likely(((__pyx_t_5) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_5, __pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 24, __pyx_L1_error) __pyx_t_7 = ((PyArrayObject *)__pyx_t_5); { __Pyx_BufFmt_StackElem __pyx_stack[1]; if (unlikely(__Pyx_GetBufferAndValidate(&__pyx_pybuffernd_scores.rcbuffer->pybuffer, (PyObject*)__pyx_t_7, &__Pyx_TypeInfo_nn___pyx_t_5numpy_float32_t, PyBUF_FORMAT| PyBUF_STRIDES, 1, 0, __pyx_stack) == -1)) { __pyx_v_scores = ((PyArrayObject *)Py_None); __Pyx_INCREF(Py_None); __pyx_pybuffernd_scores.rcbuffer->pybuffer.buf = NULL; __PYX_ERR(0, 23, __pyx_L1_error) } else {__pyx_pybuffernd_scores.diminfo[0].strides = __pyx_pybuffernd_scores.rcbuffer->pybuffer.strides[0]; __pyx_pybuffernd_scores.diminfo[0].shape = __pyx_pybuffernd_scores.rcbuffer->pybuffer.shape[0]; } } __pyx_t_7 = 0; __pyx_v_scores = ((PyArrayObject *)__pyx_t_5); __pyx_t_5 = 0; /* "nms/gpu_nms.pyx":28 * # order = scores.argsort()[::-1] * cdef np.ndarray[np.intp_t, ndim=1] \ * order = scores.argsort()[::-1] # <<<<<<<<<<<<<< * cdef np.ndarray[np.float32_t, ndim=2] \ * sorted_dets = dets[order, :] */ __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_scores), __pyx_n_s_argsort); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 28, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_3 = NULL; if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_1))) { __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_1); if (likely(__pyx_t_3)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_1); __Pyx_INCREF(__pyx_t_3); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_1, function); } } if (__pyx_t_3) { __pyx_t_5 = __Pyx_PyObject_CallOneArg(__pyx_t_1, __pyx_t_3); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 28, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else { __pyx_t_5 = __Pyx_PyObject_CallNoArg(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 28, __pyx_L1_error) } __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_1 = PyObject_GetItem(__pyx_t_5, __pyx_slice__3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 28, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 28, __pyx_L1_error) __pyx_t_8 = ((PyArrayObject *)__pyx_t_1); { __Pyx_BufFmt_StackElem __pyx_stack[1]; if (unlikely(__Pyx_GetBufferAndValidate(&__pyx_pybuffernd_order.rcbuffer->pybuffer, (PyObject*)__pyx_t_8, &__Pyx_TypeInfo_nn___pyx_t_5numpy_intp_t, PyBUF_FORMAT| PyBUF_STRIDES, 1, 0, __pyx_stack) == -1)) { __pyx_v_order = ((PyArrayObject *)Py_None); __Pyx_INCREF(Py_None); __pyx_pybuffernd_order.rcbuffer->pybuffer.buf = NULL; __PYX_ERR(0, 27, __pyx_L1_error) } else {__pyx_pybuffernd_order.diminfo[0].strides = __pyx_pybuffernd_order.rcbuffer->pybuffer.strides[0]; __pyx_pybuffernd_order.diminfo[0].shape = __pyx_pybuffernd_order.rcbuffer->pybuffer.shape[0]; } } __pyx_t_8 = 0; __pyx_v_order = ((PyArrayObject *)__pyx_t_1); __pyx_t_1 = 0; /* "nms/gpu_nms.pyx":30 * order = scores.argsort()[::-1] * cdef np.ndarray[np.float32_t, ndim=2] \ * sorted_dets = dets[order, :] # <<<<<<<<<<<<<< * _nms(&keep[0], &num_out, &sorted_dets[0, 0], boxes_num, boxes_dim, thresh, device_id) * keep = keep[:num_out] */ __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 30, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(((PyObject *)__pyx_v_order)); __Pyx_GIVEREF(((PyObject *)__pyx_v_order)); PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)__pyx_v_order)); __Pyx_INCREF(__pyx_slice__4); __Pyx_GIVEREF(__pyx_slice__4); PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_slice__4); __pyx_t_5 = PyObject_GetItem(((PyObject *)__pyx_v_dets), __pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 30, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (!(likely(((__pyx_t_5) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_5, __pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 30, __pyx_L1_error) __pyx_t_9 = ((PyArrayObject *)__pyx_t_5); { __Pyx_BufFmt_StackElem __pyx_stack[1]; if (unlikely(__Pyx_GetBufferAndValidate(&__pyx_pybuffernd_sorted_dets.rcbuffer->pybuffer, (PyObject*)__pyx_t_9, &__Pyx_TypeInfo_nn___pyx_t_5numpy_float32_t, PyBUF_FORMAT| PyBUF_STRIDES, 2, 0, __pyx_stack) == -1)) { __pyx_v_sorted_dets = ((PyArrayObject *)Py_None); __Pyx_INCREF(Py_None); __pyx_pybuffernd_sorted_dets.rcbuffer->pybuffer.buf = NULL; __PYX_ERR(0, 29, __pyx_L1_error) } else {__pyx_pybuffernd_sorted_dets.diminfo[0].strides = __pyx_pybuffernd_sorted_dets.rcbuffer->pybuffer.strides[0]; __pyx_pybuffernd_sorted_dets.diminfo[0].shape = __pyx_pybuffernd_sorted_dets.rcbuffer->pybuffer.shape[0]; __pyx_pybuffernd_sorted_dets.diminfo[1].strides = __pyx_pybuffernd_sorted_dets.rcbuffer->pybuffer.strides[1]; __pyx_pybuffernd_sorted_dets.diminfo[1].shape = __pyx_pybuffernd_sorted_dets.rcbuffer->pybuffer.shape[1]; } } __pyx_t_9 = 0; __pyx_v_sorted_dets = ((PyArrayObject *)__pyx_t_5); __pyx_t_5 = 0; /* "nms/gpu_nms.pyx":31 * cdef np.ndarray[np.float32_t, ndim=2] \ * sorted_dets = dets[order, :] * _nms(&keep[0], &num_out, &sorted_dets[0, 0], boxes_num, boxes_dim, thresh, device_id) # <<<<<<<<<<<<<< * keep = keep[:num_out] * return list(order[keep]) */ __pyx_t_10 = 0; __pyx_t_11 = -1; if (__pyx_t_10 < 0) { __pyx_t_10 += __pyx_pybuffernd_keep.diminfo[0].shape; if (unlikely(__pyx_t_10 < 0)) __pyx_t_11 = 0; } else if (unlikely(__pyx_t_10 >= __pyx_pybuffernd_keep.diminfo[0].shape)) __pyx_t_11 = 0; if (unlikely(__pyx_t_11 != -1)) { __Pyx_RaiseBufferIndexError(__pyx_t_11); __PYX_ERR(0, 31, __pyx_L1_error) } __pyx_t_12 = 0; __pyx_t_13 = 0; __pyx_t_11 = -1; if (__pyx_t_12 < 0) { __pyx_t_12 += __pyx_pybuffernd_sorted_dets.diminfo[0].shape; if (unlikely(__pyx_t_12 < 0)) __pyx_t_11 = 0; } else if (unlikely(__pyx_t_12 >= __pyx_pybuffernd_sorted_dets.diminfo[0].shape)) __pyx_t_11 = 0; if (__pyx_t_13 < 0) { __pyx_t_13 += __pyx_pybuffernd_sorted_dets.diminfo[1].shape; if (unlikely(__pyx_t_13 < 0)) __pyx_t_11 = 1; } else if (unlikely(__pyx_t_13 >= __pyx_pybuffernd_sorted_dets.diminfo[1].shape)) __pyx_t_11 = 1; if (unlikely(__pyx_t_11 != -1)) { __Pyx_RaiseBufferIndexError(__pyx_t_11); __PYX_ERR(0, 31, __pyx_L1_error) } __pyx_t_14 = __pyx_PyFloat_AsFloat(__pyx_v_thresh); if (unlikely((__pyx_t_14 == (float)-1) && PyErr_Occurred())) __PYX_ERR(0, 31, __pyx_L1_error) _nms((&(*__Pyx_BufPtrStrided1d(__pyx_t_5numpy_int32_t *, __pyx_pybuffernd_keep.rcbuffer->pybuffer.buf, __pyx_t_10, __pyx_pybuffernd_keep.diminfo[0].strides))), (&__pyx_v_num_out), (&(*__Pyx_BufPtrStrided2d(__pyx_t_5numpy_float32_t *, __pyx_pybuffernd_sorted_dets.rcbuffer->pybuffer.buf, __pyx_t_12, __pyx_pybuffernd_sorted_dets.diminfo[0].strides, __pyx_t_13, __pyx_pybuffernd_sorted_dets.diminfo[1].strides))), __pyx_v_boxes_num, __pyx_v_boxes_dim, __pyx_t_14, __pyx_v_device_id); /* "nms/gpu_nms.pyx":32 * sorted_dets = dets[order, :] * _nms(&keep[0], &num_out, &sorted_dets[0, 0], boxes_num, boxes_dim, thresh, device_id) * keep = keep[:num_out] # <<<<<<<<<<<<<< * return list(order[keep]) */ __pyx_t_5 = __Pyx_PyObject_GetSlice(((PyObject *)__pyx_v_keep), 0, __pyx_v_num_out, NULL, NULL, NULL, 0, 1, 1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 32, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); if (!(likely(((__pyx_t_5) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_5, __pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 32, __pyx_L1_error) __pyx_t_6 = ((PyArrayObject *)__pyx_t_5); { __Pyx_BufFmt_StackElem __pyx_stack[1]; __Pyx_SafeReleaseBuffer(&__pyx_pybuffernd_keep.rcbuffer->pybuffer); __pyx_t_11 = __Pyx_GetBufferAndValidate(&__pyx_pybuffernd_keep.rcbuffer->pybuffer, (PyObject*)__pyx_t_6, &__Pyx_TypeInfo_nn___pyx_t_5numpy_int32_t, PyBUF_FORMAT| PyBUF_STRIDES, 1, 0, __pyx_stack); if (unlikely(__pyx_t_11 < 0)) { PyErr_Fetch(&__pyx_t_15, &__pyx_t_16, &__pyx_t_17); if (unlikely(__Pyx_GetBufferAndValidate(&__pyx_pybuffernd_keep.rcbuffer->pybuffer, (PyObject*)__pyx_v_keep, &__Pyx_TypeInfo_nn___pyx_t_5numpy_int32_t, PyBUF_FORMAT| PyBUF_STRIDES, 1, 0, __pyx_stack) == -1)) { Py_XDECREF(__pyx_t_15); Py_XDECREF(__pyx_t_16); Py_XDECREF(__pyx_t_17); __Pyx_RaiseBufferFallbackError(); } else { PyErr_Restore(__pyx_t_15, __pyx_t_16, __pyx_t_17); } } __pyx_pybuffernd_keep.diminfo[0].strides = __pyx_pybuffernd_keep.rcbuffer->pybuffer.strides[0]; __pyx_pybuffernd_keep.diminfo[0].shape = __pyx_pybuffernd_keep.rcbuffer->pybuffer.shape[0]; if (unlikely(__pyx_t_11 < 0)) __PYX_ERR(0, 32, __pyx_L1_error) } __pyx_t_6 = 0; __Pyx_DECREF_SET(__pyx_v_keep, ((PyArrayObject *)__pyx_t_5)); __pyx_t_5 = 0; /* "nms/gpu_nms.pyx":33 * _nms(&keep[0], &num_out, &sorted_dets[0, 0], boxes_num, boxes_dim, thresh, device_id) * keep = keep[:num_out] * return list(order[keep]) # <<<<<<<<<<<<<< */ __Pyx_XDECREF(__pyx_r); __pyx_t_5 = PyObject_GetItem(((PyObject *)__pyx_v_order), ((PyObject *)__pyx_v_keep)); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 33, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_1 = PySequence_List(__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 33, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; /* "nms/gpu_nms.pyx":16 * void _nms(np.int32_t*, int*, np.float32_t*, int, int, float, int) * * def gpu_nms(np.ndarray[np.float32_t, ndim=2] dets, np.float thresh, # <<<<<<<<<<<<<< * np.int32_t device_id=0): * cdef int boxes_num = dets.shape[0] */ /* function exit code */ __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); __Pyx_XDECREF(__pyx_t_2); __Pyx_XDECREF(__pyx_t_3); __Pyx_XDECREF(__pyx_t_4); __Pyx_XDECREF(__pyx_t_5); { PyObject *__pyx_type, *__pyx_value, *__pyx_tb; __Pyx_PyThreadState_declare __Pyx_PyThreadState_assign __Pyx_ErrFetch(&__pyx_type, &__pyx_value, &__pyx_tb); __Pyx_SafeReleaseBuffer(&__pyx_pybuffernd_dets.rcbuffer->pybuffer); __Pyx_SafeReleaseBuffer(&__pyx_pybuffernd_keep.rcbuffer->pybuffer); __Pyx_SafeReleaseBuffer(&__pyx_pybuffernd_order.rcbuffer->pybuffer); __Pyx_SafeReleaseBuffer(&__pyx_pybuffernd_scores.rcbuffer->pybuffer); __Pyx_SafeReleaseBuffer(&__pyx_pybuffernd_sorted_dets.rcbuffer->pybuffer); __Pyx_ErrRestore(__pyx_type, __pyx_value, __pyx_tb);} __Pyx_AddTraceback("nms.gpu_nms.gpu_nms", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; goto __pyx_L2; __pyx_L0:; __Pyx_SafeReleaseBuffer(&__pyx_pybuffernd_dets.rcbuffer->pybuffer); __Pyx_SafeReleaseBuffer(&__pyx_pybuffernd_keep.rcbuffer->pybuffer); __Pyx_SafeReleaseBuffer(&__pyx_pybuffernd_order.rcbuffer->pybuffer); __Pyx_SafeReleaseBuffer(&__pyx_pybuffernd_scores.rcbuffer->pybuffer); __Pyx_SafeReleaseBuffer(&__pyx_pybuffernd_sorted_dets.rcbuffer->pybuffer); __pyx_L2:; __Pyx_XDECREF((PyObject *)__pyx_v_keep); __Pyx_XDECREF((PyObject *)__pyx_v_scores); __Pyx_XDECREF((PyObject *)__pyx_v_order); __Pyx_XDECREF((PyObject *)__pyx_v_sorted_dets); __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":197 * # experimental exception made for __getbuffer__ and __releasebuffer__ * # -- the details of this may change. * def __getbuffer__(ndarray self, Py_buffer* info, int flags): # <<<<<<<<<<<<<< * # This implementation of getbuffer is geared towards Cython * # requirements, and does not yet fullfill the PEP. */ /* Python wrapper */ static CYTHON_UNUSED int __pyx_pw_5numpy_7ndarray_1__getbuffer__(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /*proto*/ static CYTHON_UNUSED int __pyx_pw_5numpy_7ndarray_1__getbuffer__(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { int __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__getbuffer__ (wrapper)", 0); __pyx_r = __pyx_pf_5numpy_7ndarray___getbuffer__(((PyArrayObject *)__pyx_v_self), ((Py_buffer *)__pyx_v_info), ((int)__pyx_v_flags)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { int __pyx_v_copy_shape; int __pyx_v_i; int __pyx_v_ndim; int __pyx_v_endian_detector; int __pyx_v_little_endian; int __pyx_v_t; char *__pyx_v_f; PyArray_Descr *__pyx_v_descr = 0; int __pyx_v_offset; int __pyx_v_hasfields; int __pyx_r; __Pyx_RefNannyDeclarations int __pyx_t_1; int __pyx_t_2; PyObject *__pyx_t_3 = NULL; int __pyx_t_4; int __pyx_t_5; PyObject *__pyx_t_6 = NULL; char *__pyx_t_7; __Pyx_RefNannySetupContext("__getbuffer__", 0); if (__pyx_v_info != NULL) { __pyx_v_info->obj = Py_None; __Pyx_INCREF(Py_None); __Pyx_GIVEREF(__pyx_v_info->obj); } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":203 * # of flags * * if info == NULL: return # <<<<<<<<<<<<<< * * cdef int copy_shape, i, ndim */ __pyx_t_1 = ((__pyx_v_info == NULL) != 0); if (__pyx_t_1) { __pyx_r = 0; goto __pyx_L0; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":206 * * cdef int copy_shape, i, ndim * cdef int endian_detector = 1 # <<<<<<<<<<<<<< * cdef bint little_endian = ((&endian_detector)[0] != 0) * */ __pyx_v_endian_detector = 1; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":207 * cdef int copy_shape, i, ndim * cdef int endian_detector = 1 * cdef bint little_endian = ((&endian_detector)[0] != 0) # <<<<<<<<<<<<<< * * ndim = PyArray_NDIM(self) */ __pyx_v_little_endian = ((((char *)(&__pyx_v_endian_detector))[0]) != 0); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":209 * cdef bint little_endian = ((&endian_detector)[0] != 0) * * ndim = PyArray_NDIM(self) # <<<<<<<<<<<<<< * * if sizeof(npy_intp) != sizeof(Py_ssize_t): */ __pyx_v_ndim = PyArray_NDIM(__pyx_v_self); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":211 * ndim = PyArray_NDIM(self) * * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< * copy_shape = 1 * else: */ __pyx_t_1 = (((sizeof(npy_intp)) != (sizeof(Py_ssize_t))) != 0); if (__pyx_t_1) { /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":212 * * if sizeof(npy_intp) != sizeof(Py_ssize_t): * copy_shape = 1 # <<<<<<<<<<<<<< * else: * copy_shape = 0 */ __pyx_v_copy_shape = 1; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":211 * ndim = PyArray_NDIM(self) * * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< * copy_shape = 1 * else: */ goto __pyx_L4; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":214 * copy_shape = 1 * else: * copy_shape = 0 # <<<<<<<<<<<<<< * * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) */ /*else*/ { __pyx_v_copy_shape = 0; } __pyx_L4:; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":216 * copy_shape = 0 * * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< * and not PyArray_CHKFLAGS(self, NPY_C_CONTIGUOUS)): * raise ValueError(u"ndarray is not C contiguous") */ __pyx_t_2 = (((__pyx_v_flags & PyBUF_C_CONTIGUOUS) == PyBUF_C_CONTIGUOUS) != 0); if (__pyx_t_2) { } else { __pyx_t_1 = __pyx_t_2; goto __pyx_L6_bool_binop_done; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":217 * * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_C_CONTIGUOUS)): # <<<<<<<<<<<<<< * raise ValueError(u"ndarray is not C contiguous") * */ __pyx_t_2 = ((!(PyArray_CHKFLAGS(__pyx_v_self, NPY_C_CONTIGUOUS) != 0)) != 0); __pyx_t_1 = __pyx_t_2; __pyx_L6_bool_binop_done:; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":216 * copy_shape = 0 * * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< * and not PyArray_CHKFLAGS(self, NPY_C_CONTIGUOUS)): * raise ValueError(u"ndarray is not C contiguous") */ if (__pyx_t_1) { /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":218 * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_C_CONTIGUOUS)): * raise ValueError(u"ndarray is not C contiguous") # <<<<<<<<<<<<<< * * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) */ __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__5, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 218, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 218, __pyx_L1_error) /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":216 * copy_shape = 0 * * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) # <<<<<<<<<<<<<< * and not PyArray_CHKFLAGS(self, NPY_C_CONTIGUOUS)): * raise ValueError(u"ndarray is not C contiguous") */ } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":220 * raise ValueError(u"ndarray is not C contiguous") * * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) # <<<<<<<<<<<<<< * and not PyArray_CHKFLAGS(self, NPY_F_CONTIGUOUS)): * raise ValueError(u"ndarray is not Fortran contiguous") */ __pyx_t_2 = (((__pyx_v_flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS) != 0); if (__pyx_t_2) { } else { __pyx_t_1 = __pyx_t_2; goto __pyx_L9_bool_binop_done; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":221 * * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_F_CONTIGUOUS)): # <<<<<<<<<<<<<< * raise ValueError(u"ndarray is not Fortran contiguous") * */ __pyx_t_2 = ((!(PyArray_CHKFLAGS(__pyx_v_self, NPY_F_CONTIGUOUS) != 0)) != 0); __pyx_t_1 = __pyx_t_2; __pyx_L9_bool_binop_done:; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":220 * raise ValueError(u"ndarray is not C contiguous") * * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) # <<<<<<<<<<<<<< * and not PyArray_CHKFLAGS(self, NPY_F_CONTIGUOUS)): * raise ValueError(u"ndarray is not Fortran contiguous") */ if (__pyx_t_1) { /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":222 * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_F_CONTIGUOUS)): * raise ValueError(u"ndarray is not Fortran contiguous") # <<<<<<<<<<<<<< * * info.buf = PyArray_DATA(self) */ __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__6, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 222, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 222, __pyx_L1_error) /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":220 * raise ValueError(u"ndarray is not C contiguous") * * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) # <<<<<<<<<<<<<< * and not PyArray_CHKFLAGS(self, NPY_F_CONTIGUOUS)): * raise ValueError(u"ndarray is not Fortran contiguous") */ } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":224 * raise ValueError(u"ndarray is not Fortran contiguous") * * info.buf = PyArray_DATA(self) # <<<<<<<<<<<<<< * info.ndim = ndim * if copy_shape: */ __pyx_v_info->buf = PyArray_DATA(__pyx_v_self); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":225 * * info.buf = PyArray_DATA(self) * info.ndim = ndim # <<<<<<<<<<<<<< * if copy_shape: * # Allocate new buffer for strides and shape info. */ __pyx_v_info->ndim = __pyx_v_ndim; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":226 * info.buf = PyArray_DATA(self) * info.ndim = ndim * if copy_shape: # <<<<<<<<<<<<<< * # Allocate new buffer for strides and shape info. * # This is allocated as one block, strides first. */ __pyx_t_1 = (__pyx_v_copy_shape != 0); if (__pyx_t_1) { /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":229 * # Allocate new buffer for strides and shape info. * # This is allocated as one block, strides first. * info.strides = stdlib.malloc(sizeof(Py_ssize_t) * ndim * 2) # <<<<<<<<<<<<<< * info.shape = info.strides + ndim * for i in range(ndim): */ __pyx_v_info->strides = ((Py_ssize_t *)malloc((((sizeof(Py_ssize_t)) * ((size_t)__pyx_v_ndim)) * 2))); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":230 * # This is allocated as one block, strides first. * info.strides = stdlib.malloc(sizeof(Py_ssize_t) * ndim * 2) * info.shape = info.strides + ndim # <<<<<<<<<<<<<< * for i in range(ndim): * info.strides[i] = PyArray_STRIDES(self)[i] */ __pyx_v_info->shape = (__pyx_v_info->strides + __pyx_v_ndim); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":231 * info.strides = stdlib.malloc(sizeof(Py_ssize_t) * ndim * 2) * info.shape = info.strides + ndim * for i in range(ndim): # <<<<<<<<<<<<<< * info.strides[i] = PyArray_STRIDES(self)[i] * info.shape[i] = PyArray_DIMS(self)[i] */ __pyx_t_4 = __pyx_v_ndim; for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_4; __pyx_t_5+=1) { __pyx_v_i = __pyx_t_5; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":232 * info.shape = info.strides + ndim * for i in range(ndim): * info.strides[i] = PyArray_STRIDES(self)[i] # <<<<<<<<<<<<<< * info.shape[i] = PyArray_DIMS(self)[i] * else: */ (__pyx_v_info->strides[__pyx_v_i]) = (PyArray_STRIDES(__pyx_v_self)[__pyx_v_i]); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":233 * for i in range(ndim): * info.strides[i] = PyArray_STRIDES(self)[i] * info.shape[i] = PyArray_DIMS(self)[i] # <<<<<<<<<<<<<< * else: * info.strides = PyArray_STRIDES(self) */ (__pyx_v_info->shape[__pyx_v_i]) = (PyArray_DIMS(__pyx_v_self)[__pyx_v_i]); } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":226 * info.buf = PyArray_DATA(self) * info.ndim = ndim * if copy_shape: # <<<<<<<<<<<<<< * # Allocate new buffer for strides and shape info. * # This is allocated as one block, strides first. */ goto __pyx_L11; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":235 * info.shape[i] = PyArray_DIMS(self)[i] * else: * info.strides = PyArray_STRIDES(self) # <<<<<<<<<<<<<< * info.shape = PyArray_DIMS(self) * info.suboffsets = NULL */ /*else*/ { __pyx_v_info->strides = ((Py_ssize_t *)PyArray_STRIDES(__pyx_v_self)); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":236 * else: * info.strides = PyArray_STRIDES(self) * info.shape = PyArray_DIMS(self) # <<<<<<<<<<<<<< * info.suboffsets = NULL * info.itemsize = PyArray_ITEMSIZE(self) */ __pyx_v_info->shape = ((Py_ssize_t *)PyArray_DIMS(__pyx_v_self)); } __pyx_L11:; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":237 * info.strides = PyArray_STRIDES(self) * info.shape = PyArray_DIMS(self) * info.suboffsets = NULL # <<<<<<<<<<<<<< * info.itemsize = PyArray_ITEMSIZE(self) * info.readonly = not PyArray_ISWRITEABLE(self) */ __pyx_v_info->suboffsets = NULL; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":238 * info.shape = PyArray_DIMS(self) * info.suboffsets = NULL * info.itemsize = PyArray_ITEMSIZE(self) # <<<<<<<<<<<<<< * info.readonly = not PyArray_ISWRITEABLE(self) * */ __pyx_v_info->itemsize = PyArray_ITEMSIZE(__pyx_v_self); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":239 * info.suboffsets = NULL * info.itemsize = PyArray_ITEMSIZE(self) * info.readonly = not PyArray_ISWRITEABLE(self) # <<<<<<<<<<<<<< * * cdef int t */ __pyx_v_info->readonly = (!(PyArray_ISWRITEABLE(__pyx_v_self) != 0)); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":242 * * cdef int t * cdef char* f = NULL # <<<<<<<<<<<<<< * cdef dtype descr = self.descr * cdef int offset */ __pyx_v_f = NULL; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":243 * cdef int t * cdef char* f = NULL * cdef dtype descr = self.descr # <<<<<<<<<<<<<< * cdef int offset * */ __pyx_t_3 = ((PyObject *)__pyx_v_self->descr); __Pyx_INCREF(__pyx_t_3); __pyx_v_descr = ((PyArray_Descr *)__pyx_t_3); __pyx_t_3 = 0; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":246 * cdef int offset * * cdef bint hasfields = PyDataType_HASFIELDS(descr) # <<<<<<<<<<<<<< * * if not hasfields and not copy_shape: */ __pyx_v_hasfields = PyDataType_HASFIELDS(__pyx_v_descr); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":248 * cdef bint hasfields = PyDataType_HASFIELDS(descr) * * if not hasfields and not copy_shape: # <<<<<<<<<<<<<< * # do not call releasebuffer * info.obj = None */ __pyx_t_2 = ((!(__pyx_v_hasfields != 0)) != 0); if (__pyx_t_2) { } else { __pyx_t_1 = __pyx_t_2; goto __pyx_L15_bool_binop_done; } __pyx_t_2 = ((!(__pyx_v_copy_shape != 0)) != 0); __pyx_t_1 = __pyx_t_2; __pyx_L15_bool_binop_done:; if (__pyx_t_1) { /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":250 * if not hasfields and not copy_shape: * # do not call releasebuffer * info.obj = None # <<<<<<<<<<<<<< * else: * # need to call releasebuffer */ __Pyx_INCREF(Py_None); __Pyx_GIVEREF(Py_None); __Pyx_GOTREF(__pyx_v_info->obj); __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = Py_None; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":248 * cdef bint hasfields = PyDataType_HASFIELDS(descr) * * if not hasfields and not copy_shape: # <<<<<<<<<<<<<< * # do not call releasebuffer * info.obj = None */ goto __pyx_L14; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":253 * else: * # need to call releasebuffer * info.obj = self # <<<<<<<<<<<<<< * * if not hasfields: */ /*else*/ { __Pyx_INCREF(((PyObject *)__pyx_v_self)); __Pyx_GIVEREF(((PyObject *)__pyx_v_self)); __Pyx_GOTREF(__pyx_v_info->obj); __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = ((PyObject *)__pyx_v_self); } __pyx_L14:; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":255 * info.obj = self * * if not hasfields: # <<<<<<<<<<<<<< * t = descr.type_num * if ((descr.byteorder == c'>' and little_endian) or */ __pyx_t_1 = ((!(__pyx_v_hasfields != 0)) != 0); if (__pyx_t_1) { /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":256 * * if not hasfields: * t = descr.type_num # <<<<<<<<<<<<<< * if ((descr.byteorder == c'>' and little_endian) or * (descr.byteorder == c'<' and not little_endian)): */ __pyx_t_4 = __pyx_v_descr->type_num; __pyx_v_t = __pyx_t_4; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":257 * if not hasfields: * t = descr.type_num * if ((descr.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< * (descr.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") */ __pyx_t_2 = ((__pyx_v_descr->byteorder == '>') != 0); if (!__pyx_t_2) { goto __pyx_L20_next_or; } else { } __pyx_t_2 = (__pyx_v_little_endian != 0); if (!__pyx_t_2) { } else { __pyx_t_1 = __pyx_t_2; goto __pyx_L19_bool_binop_done; } __pyx_L20_next_or:; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":258 * t = descr.type_num * if ((descr.byteorder == c'>' and little_endian) or * (descr.byteorder == c'<' and not little_endian)): # <<<<<<<<<<<<<< * raise ValueError(u"Non-native byte order not supported") * if t == NPY_BYTE: f = "b" */ __pyx_t_2 = ((__pyx_v_descr->byteorder == '<') != 0); if (__pyx_t_2) { } else { __pyx_t_1 = __pyx_t_2; goto __pyx_L19_bool_binop_done; } __pyx_t_2 = ((!(__pyx_v_little_endian != 0)) != 0); __pyx_t_1 = __pyx_t_2; __pyx_L19_bool_binop_done:; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":257 * if not hasfields: * t = descr.type_num * if ((descr.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< * (descr.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") */ if (__pyx_t_1) { /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":259 * if ((descr.byteorder == c'>' and little_endian) or * (descr.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") # <<<<<<<<<<<<<< * if t == NPY_BYTE: f = "b" * elif t == NPY_UBYTE: f = "B" */ __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__7, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 259, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 259, __pyx_L1_error) /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":257 * if not hasfields: * t = descr.type_num * if ((descr.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< * (descr.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") */ } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":260 * (descr.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") * if t == NPY_BYTE: f = "b" # <<<<<<<<<<<<<< * elif t == NPY_UBYTE: f = "B" * elif t == NPY_SHORT: f = "h" */ switch (__pyx_v_t) { case NPY_BYTE: __pyx_v_f = ((char *)"b"); break; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":261 * raise ValueError(u"Non-native byte order not supported") * if t == NPY_BYTE: f = "b" * elif t == NPY_UBYTE: f = "B" # <<<<<<<<<<<<<< * elif t == NPY_SHORT: f = "h" * elif t == NPY_USHORT: f = "H" */ case NPY_UBYTE: __pyx_v_f = ((char *)"B"); break; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":262 * if t == NPY_BYTE: f = "b" * elif t == NPY_UBYTE: f = "B" * elif t == NPY_SHORT: f = "h" # <<<<<<<<<<<<<< * elif t == NPY_USHORT: f = "H" * elif t == NPY_INT: f = "i" */ case NPY_SHORT: __pyx_v_f = ((char *)"h"); break; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":263 * elif t == NPY_UBYTE: f = "B" * elif t == NPY_SHORT: f = "h" * elif t == NPY_USHORT: f = "H" # <<<<<<<<<<<<<< * elif t == NPY_INT: f = "i" * elif t == NPY_UINT: f = "I" */ case NPY_USHORT: __pyx_v_f = ((char *)"H"); break; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":264 * elif t == NPY_SHORT: f = "h" * elif t == NPY_USHORT: f = "H" * elif t == NPY_INT: f = "i" # <<<<<<<<<<<<<< * elif t == NPY_UINT: f = "I" * elif t == NPY_LONG: f = "l" */ case NPY_INT: __pyx_v_f = ((char *)"i"); break; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":265 * elif t == NPY_USHORT: f = "H" * elif t == NPY_INT: f = "i" * elif t == NPY_UINT: f = "I" # <<<<<<<<<<<<<< * elif t == NPY_LONG: f = "l" * elif t == NPY_ULONG: f = "L" */ case NPY_UINT: __pyx_v_f = ((char *)"I"); break; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":266 * elif t == NPY_INT: f = "i" * elif t == NPY_UINT: f = "I" * elif t == NPY_LONG: f = "l" # <<<<<<<<<<<<<< * elif t == NPY_ULONG: f = "L" * elif t == NPY_LONGLONG: f = "q" */ case NPY_LONG: __pyx_v_f = ((char *)"l"); break; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":267 * elif t == NPY_UINT: f = "I" * elif t == NPY_LONG: f = "l" * elif t == NPY_ULONG: f = "L" # <<<<<<<<<<<<<< * elif t == NPY_LONGLONG: f = "q" * elif t == NPY_ULONGLONG: f = "Q" */ case NPY_ULONG: __pyx_v_f = ((char *)"L"); break; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":268 * elif t == NPY_LONG: f = "l" * elif t == NPY_ULONG: f = "L" * elif t == NPY_LONGLONG: f = "q" # <<<<<<<<<<<<<< * elif t == NPY_ULONGLONG: f = "Q" * elif t == NPY_FLOAT: f = "f" */ case NPY_LONGLONG: __pyx_v_f = ((char *)"q"); break; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":269 * elif t == NPY_ULONG: f = "L" * elif t == NPY_LONGLONG: f = "q" * elif t == NPY_ULONGLONG: f = "Q" # <<<<<<<<<<<<<< * elif t == NPY_FLOAT: f = "f" * elif t == NPY_DOUBLE: f = "d" */ case NPY_ULONGLONG: __pyx_v_f = ((char *)"Q"); break; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":270 * elif t == NPY_LONGLONG: f = "q" * elif t == NPY_ULONGLONG: f = "Q" * elif t == NPY_FLOAT: f = "f" # <<<<<<<<<<<<<< * elif t == NPY_DOUBLE: f = "d" * elif t == NPY_LONGDOUBLE: f = "g" */ case NPY_FLOAT: __pyx_v_f = ((char *)"f"); break; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":271 * elif t == NPY_ULONGLONG: f = "Q" * elif t == NPY_FLOAT: f = "f" * elif t == NPY_DOUBLE: f = "d" # <<<<<<<<<<<<<< * elif t == NPY_LONGDOUBLE: f = "g" * elif t == NPY_CFLOAT: f = "Zf" */ case NPY_DOUBLE: __pyx_v_f = ((char *)"d"); break; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":272 * elif t == NPY_FLOAT: f = "f" * elif t == NPY_DOUBLE: f = "d" * elif t == NPY_LONGDOUBLE: f = "g" # <<<<<<<<<<<<<< * elif t == NPY_CFLOAT: f = "Zf" * elif t == NPY_CDOUBLE: f = "Zd" */ case NPY_LONGDOUBLE: __pyx_v_f = ((char *)"g"); break; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":273 * elif t == NPY_DOUBLE: f = "d" * elif t == NPY_LONGDOUBLE: f = "g" * elif t == NPY_CFLOAT: f = "Zf" # <<<<<<<<<<<<<< * elif t == NPY_CDOUBLE: f = "Zd" * elif t == NPY_CLONGDOUBLE: f = "Zg" */ case NPY_CFLOAT: __pyx_v_f = ((char *)"Zf"); break; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":274 * elif t == NPY_LONGDOUBLE: f = "g" * elif t == NPY_CFLOAT: f = "Zf" * elif t == NPY_CDOUBLE: f = "Zd" # <<<<<<<<<<<<<< * elif t == NPY_CLONGDOUBLE: f = "Zg" * elif t == NPY_OBJECT: f = "O" */ case NPY_CDOUBLE: __pyx_v_f = ((char *)"Zd"); break; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":275 * elif t == NPY_CFLOAT: f = "Zf" * elif t == NPY_CDOUBLE: f = "Zd" * elif t == NPY_CLONGDOUBLE: f = "Zg" # <<<<<<<<<<<<<< * elif t == NPY_OBJECT: f = "O" * else: */ case NPY_CLONGDOUBLE: __pyx_v_f = ((char *)"Zg"); break; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":276 * elif t == NPY_CDOUBLE: f = "Zd" * elif t == NPY_CLONGDOUBLE: f = "Zg" * elif t == NPY_OBJECT: f = "O" # <<<<<<<<<<<<<< * else: * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) */ case NPY_OBJECT: __pyx_v_f = ((char *)"O"); break; default: /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":278 * elif t == NPY_OBJECT: f = "O" * else: * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) # <<<<<<<<<<<<<< * info.format = f * return */ __pyx_t_3 = __Pyx_PyInt_From_int(__pyx_v_t); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 278, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_6 = PyUnicode_Format(__pyx_kp_u_unknown_dtype_code_in_numpy_pxd, __pyx_t_3); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 278, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 278, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_6); __pyx_t_6 = 0; __pyx_t_6 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_t_3, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 278, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_Raise(__pyx_t_6, 0, 0, 0); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __PYX_ERR(1, 278, __pyx_L1_error) break; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":279 * else: * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) * info.format = f # <<<<<<<<<<<<<< * return * else: */ __pyx_v_info->format = __pyx_v_f; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":280 * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) * info.format = f * return # <<<<<<<<<<<<<< * else: * info.format = stdlib.malloc(_buffer_format_string_len) */ __pyx_r = 0; goto __pyx_L0; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":255 * info.obj = self * * if not hasfields: # <<<<<<<<<<<<<< * t = descr.type_num * if ((descr.byteorder == c'>' and little_endian) or */ } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":282 * return * else: * info.format = stdlib.malloc(_buffer_format_string_len) # <<<<<<<<<<<<<< * info.format[0] = c'^' # Native data types, manual alignment * offset = 0 */ /*else*/ { __pyx_v_info->format = ((char *)malloc(0xFF)); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":283 * else: * info.format = stdlib.malloc(_buffer_format_string_len) * info.format[0] = c'^' # Native data types, manual alignment # <<<<<<<<<<<<<< * offset = 0 * f = _util_dtypestring(descr, info.format + 1, */ (__pyx_v_info->format[0]) = '^'; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":284 * info.format = stdlib.malloc(_buffer_format_string_len) * info.format[0] = c'^' # Native data types, manual alignment * offset = 0 # <<<<<<<<<<<<<< * f = _util_dtypestring(descr, info.format + 1, * info.format + _buffer_format_string_len, */ __pyx_v_offset = 0; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":285 * info.format[0] = c'^' # Native data types, manual alignment * offset = 0 * f = _util_dtypestring(descr, info.format + 1, # <<<<<<<<<<<<<< * info.format + _buffer_format_string_len, * &offset) */ __pyx_t_7 = __pyx_f_5numpy__util_dtypestring(__pyx_v_descr, (__pyx_v_info->format + 1), (__pyx_v_info->format + 0xFF), (&__pyx_v_offset)); if (unlikely(__pyx_t_7 == NULL)) __PYX_ERR(1, 285, __pyx_L1_error) __pyx_v_f = __pyx_t_7; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":288 * info.format + _buffer_format_string_len, * &offset) * f[0] = c'\0' # Terminate format string # <<<<<<<<<<<<<< * * def __releasebuffer__(ndarray self, Py_buffer* info): */ (__pyx_v_f[0]) = '\x00'; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":197 * # experimental exception made for __getbuffer__ and __releasebuffer__ * # -- the details of this may change. * def __getbuffer__(ndarray self, Py_buffer* info, int flags): # <<<<<<<<<<<<<< * # This implementation of getbuffer is geared towards Cython * # requirements, and does not yet fullfill the PEP. */ /* function exit code */ __pyx_r = 0; goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_3); __Pyx_XDECREF(__pyx_t_6); __Pyx_AddTraceback("numpy.ndarray.__getbuffer__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = -1; if (__pyx_v_info != NULL && __pyx_v_info->obj != NULL) { __Pyx_GOTREF(__pyx_v_info->obj); __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = NULL; } goto __pyx_L2; __pyx_L0:; if (__pyx_v_info != NULL && __pyx_v_info->obj == Py_None) { __Pyx_GOTREF(Py_None); __Pyx_DECREF(Py_None); __pyx_v_info->obj = NULL; } __pyx_L2:; __Pyx_XDECREF((PyObject *)__pyx_v_descr); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":290 * f[0] = c'\0' # Terminate format string * * def __releasebuffer__(ndarray self, Py_buffer* info): # <<<<<<<<<<<<<< * if PyArray_HASFIELDS(self): * stdlib.free(info.format) */ /* Python wrapper */ static CYTHON_UNUSED void __pyx_pw_5numpy_7ndarray_3__releasebuffer__(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info); /*proto*/ static CYTHON_UNUSED void __pyx_pw_5numpy_7ndarray_3__releasebuffer__(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info) { __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__releasebuffer__ (wrapper)", 0); __pyx_pf_5numpy_7ndarray_2__releasebuffer__(((PyArrayObject *)__pyx_v_self), ((Py_buffer *)__pyx_v_info)); /* function exit code */ __Pyx_RefNannyFinishContext(); } static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info) { __Pyx_RefNannyDeclarations int __pyx_t_1; __Pyx_RefNannySetupContext("__releasebuffer__", 0); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":291 * * def __releasebuffer__(ndarray self, Py_buffer* info): * if PyArray_HASFIELDS(self): # <<<<<<<<<<<<<< * stdlib.free(info.format) * if sizeof(npy_intp) != sizeof(Py_ssize_t): */ __pyx_t_1 = (PyArray_HASFIELDS(__pyx_v_self) != 0); if (__pyx_t_1) { /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":292 * def __releasebuffer__(ndarray self, Py_buffer* info): * if PyArray_HASFIELDS(self): * stdlib.free(info.format) # <<<<<<<<<<<<<< * if sizeof(npy_intp) != sizeof(Py_ssize_t): * stdlib.free(info.strides) */ free(__pyx_v_info->format); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":291 * * def __releasebuffer__(ndarray self, Py_buffer* info): * if PyArray_HASFIELDS(self): # <<<<<<<<<<<<<< * stdlib.free(info.format) * if sizeof(npy_intp) != sizeof(Py_ssize_t): */ } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":293 * if PyArray_HASFIELDS(self): * stdlib.free(info.format) * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< * stdlib.free(info.strides) * # info.shape was stored after info.strides in the same block */ __pyx_t_1 = (((sizeof(npy_intp)) != (sizeof(Py_ssize_t))) != 0); if (__pyx_t_1) { /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":294 * stdlib.free(info.format) * if sizeof(npy_intp) != sizeof(Py_ssize_t): * stdlib.free(info.strides) # <<<<<<<<<<<<<< * # info.shape was stored after info.strides in the same block * */ free(__pyx_v_info->strides); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":293 * if PyArray_HASFIELDS(self): * stdlib.free(info.format) * if sizeof(npy_intp) != sizeof(Py_ssize_t): # <<<<<<<<<<<<<< * stdlib.free(info.strides) * # info.shape was stored after info.strides in the same block */ } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":290 * f[0] = c'\0' # Terminate format string * * def __releasebuffer__(ndarray self, Py_buffer* info): # <<<<<<<<<<<<<< * if PyArray_HASFIELDS(self): * stdlib.free(info.format) */ /* function exit code */ __Pyx_RefNannyFinishContext(); } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":770 * ctypedef npy_cdouble complex_t * * cdef inline object PyArray_MultiIterNew1(a): # <<<<<<<<<<<<<< * return PyArray_MultiIterNew(1, a) * */ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew1(PyObject *__pyx_v_a) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew1", 0); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":771 * * cdef inline object PyArray_MultiIterNew1(a): * return PyArray_MultiIterNew(1, a) # <<<<<<<<<<<<<< * * cdef inline object PyArray_MultiIterNew2(a, b): */ __Pyx_XDECREF(__pyx_r); __pyx_t_1 = PyArray_MultiIterNew(1, ((void *)__pyx_v_a)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 771, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":770 * ctypedef npy_cdouble complex_t * * cdef inline object PyArray_MultiIterNew1(a): # <<<<<<<<<<<<<< * return PyArray_MultiIterNew(1, a) * */ /* function exit code */ __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); __Pyx_AddTraceback("numpy.PyArray_MultiIterNew1", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = 0; __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":773 * return PyArray_MultiIterNew(1, a) * * cdef inline object PyArray_MultiIterNew2(a, b): # <<<<<<<<<<<<<< * return PyArray_MultiIterNew(2, a, b) * */ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew2(PyObject *__pyx_v_a, PyObject *__pyx_v_b) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew2", 0); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":774 * * cdef inline object PyArray_MultiIterNew2(a, b): * return PyArray_MultiIterNew(2, a, b) # <<<<<<<<<<<<<< * * cdef inline object PyArray_MultiIterNew3(a, b, c): */ __Pyx_XDECREF(__pyx_r); __pyx_t_1 = PyArray_MultiIterNew(2, ((void *)__pyx_v_a), ((void *)__pyx_v_b)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 774, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":773 * return PyArray_MultiIterNew(1, a) * * cdef inline object PyArray_MultiIterNew2(a, b): # <<<<<<<<<<<<<< * return PyArray_MultiIterNew(2, a, b) * */ /* function exit code */ __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); __Pyx_AddTraceback("numpy.PyArray_MultiIterNew2", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = 0; __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":776 * return PyArray_MultiIterNew(2, a, b) * * cdef inline object PyArray_MultiIterNew3(a, b, c): # <<<<<<<<<<<<<< * return PyArray_MultiIterNew(3, a, b, c) * */ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew3(PyObject *__pyx_v_a, PyObject *__pyx_v_b, PyObject *__pyx_v_c) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew3", 0); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":777 * * cdef inline object PyArray_MultiIterNew3(a, b, c): * return PyArray_MultiIterNew(3, a, b, c) # <<<<<<<<<<<<<< * * cdef inline object PyArray_MultiIterNew4(a, b, c, d): */ __Pyx_XDECREF(__pyx_r); __pyx_t_1 = PyArray_MultiIterNew(3, ((void *)__pyx_v_a), ((void *)__pyx_v_b), ((void *)__pyx_v_c)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 777, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":776 * return PyArray_MultiIterNew(2, a, b) * * cdef inline object PyArray_MultiIterNew3(a, b, c): # <<<<<<<<<<<<<< * return PyArray_MultiIterNew(3, a, b, c) * */ /* function exit code */ __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); __Pyx_AddTraceback("numpy.PyArray_MultiIterNew3", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = 0; __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":779 * return PyArray_MultiIterNew(3, a, b, c) * * cdef inline object PyArray_MultiIterNew4(a, b, c, d): # <<<<<<<<<<<<<< * return PyArray_MultiIterNew(4, a, b, c, d) * */ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew4(PyObject *__pyx_v_a, PyObject *__pyx_v_b, PyObject *__pyx_v_c, PyObject *__pyx_v_d) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew4", 0); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":780 * * cdef inline object PyArray_MultiIterNew4(a, b, c, d): * return PyArray_MultiIterNew(4, a, b, c, d) # <<<<<<<<<<<<<< * * cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): */ __Pyx_XDECREF(__pyx_r); __pyx_t_1 = PyArray_MultiIterNew(4, ((void *)__pyx_v_a), ((void *)__pyx_v_b), ((void *)__pyx_v_c), ((void *)__pyx_v_d)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 780, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":779 * return PyArray_MultiIterNew(3, a, b, c) * * cdef inline object PyArray_MultiIterNew4(a, b, c, d): # <<<<<<<<<<<<<< * return PyArray_MultiIterNew(4, a, b, c, d) * */ /* function exit code */ __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); __Pyx_AddTraceback("numpy.PyArray_MultiIterNew4", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = 0; __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":782 * return PyArray_MultiIterNew(4, a, b, c, d) * * cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): # <<<<<<<<<<<<<< * return PyArray_MultiIterNew(5, a, b, c, d, e) * */ static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew5(PyObject *__pyx_v_a, PyObject *__pyx_v_b, PyObject *__pyx_v_c, PyObject *__pyx_v_d, PyObject *__pyx_v_e) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("PyArray_MultiIterNew5", 0); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":783 * * cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): * return PyArray_MultiIterNew(5, a, b, c, d, e) # <<<<<<<<<<<<<< * * cdef inline char* _util_dtypestring(dtype descr, char* f, char* end, int* offset) except NULL: */ __Pyx_XDECREF(__pyx_r); __pyx_t_1 = PyArray_MultiIterNew(5, ((void *)__pyx_v_a), ((void *)__pyx_v_b), ((void *)__pyx_v_c), ((void *)__pyx_v_d), ((void *)__pyx_v_e)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 783, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":782 * return PyArray_MultiIterNew(4, a, b, c, d) * * cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): # <<<<<<<<<<<<<< * return PyArray_MultiIterNew(5, a, b, c, d, e) * */ /* function exit code */ __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); __Pyx_AddTraceback("numpy.PyArray_MultiIterNew5", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = 0; __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":785 * return PyArray_MultiIterNew(5, a, b, c, d, e) * * cdef inline char* _util_dtypestring(dtype descr, char* f, char* end, int* offset) except NULL: # <<<<<<<<<<<<<< * # Recursive utility function used in __getbuffer__ to get format * # string. The new location in the format string is returned. */ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *__pyx_v_descr, char *__pyx_v_f, char *__pyx_v_end, int *__pyx_v_offset) { PyArray_Descr *__pyx_v_child = 0; int __pyx_v_endian_detector; int __pyx_v_little_endian; PyObject *__pyx_v_fields = 0; PyObject *__pyx_v_childname = NULL; PyObject *__pyx_v_new_offset = NULL; PyObject *__pyx_v_t = NULL; char *__pyx_r; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; Py_ssize_t __pyx_t_2; PyObject *__pyx_t_3 = NULL; PyObject *__pyx_t_4 = NULL; int __pyx_t_5; int __pyx_t_6; int __pyx_t_7; long __pyx_t_8; char *__pyx_t_9; __Pyx_RefNannySetupContext("_util_dtypestring", 0); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":790 * * cdef dtype child * cdef int endian_detector = 1 # <<<<<<<<<<<<<< * cdef bint little_endian = ((&endian_detector)[0] != 0) * cdef tuple fields */ __pyx_v_endian_detector = 1; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":791 * cdef dtype child * cdef int endian_detector = 1 * cdef bint little_endian = ((&endian_detector)[0] != 0) # <<<<<<<<<<<<<< * cdef tuple fields * */ __pyx_v_little_endian = ((((char *)(&__pyx_v_endian_detector))[0]) != 0); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":794 * cdef tuple fields * * for childname in descr.names: # <<<<<<<<<<<<<< * fields = descr.fields[childname] * child, new_offset = fields */ if (unlikely(__pyx_v_descr->names == Py_None)) { PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); __PYX_ERR(1, 794, __pyx_L1_error) } __pyx_t_1 = __pyx_v_descr->names; __Pyx_INCREF(__pyx_t_1); __pyx_t_2 = 0; for (;;) { if (__pyx_t_2 >= PyTuple_GET_SIZE(__pyx_t_1)) break; #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_3 = PyTuple_GET_ITEM(__pyx_t_1, __pyx_t_2); __Pyx_INCREF(__pyx_t_3); __pyx_t_2++; if (unlikely(0 < 0)) __PYX_ERR(1, 794, __pyx_L1_error) #else __pyx_t_3 = PySequence_ITEM(__pyx_t_1, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 794, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); #endif __Pyx_XDECREF_SET(__pyx_v_childname, __pyx_t_3); __pyx_t_3 = 0; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":795 * * for childname in descr.names: * fields = descr.fields[childname] # <<<<<<<<<<<<<< * child, new_offset = fields * */ if (unlikely(__pyx_v_descr->fields == Py_None)) { PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); __PYX_ERR(1, 795, __pyx_L1_error) } __pyx_t_3 = __Pyx_PyDict_GetItem(__pyx_v_descr->fields, __pyx_v_childname); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 795, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); if (!(likely(PyTuple_CheckExact(__pyx_t_3))||((__pyx_t_3) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "tuple", Py_TYPE(__pyx_t_3)->tp_name), 0))) __PYX_ERR(1, 795, __pyx_L1_error) __Pyx_XDECREF_SET(__pyx_v_fields, ((PyObject*)__pyx_t_3)); __pyx_t_3 = 0; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":796 * for childname in descr.names: * fields = descr.fields[childname] * child, new_offset = fields # <<<<<<<<<<<<<< * * if (end - f) - (new_offset - offset[0]) < 15: */ if (likely(__pyx_v_fields != Py_None)) { PyObject* sequence = __pyx_v_fields; #if CYTHON_COMPILING_IN_CPYTHON Py_ssize_t size = Py_SIZE(sequence); #else Py_ssize_t size = PySequence_Size(sequence); #endif if (unlikely(size != 2)) { if (size > 2) __Pyx_RaiseTooManyValuesError(2); else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); __PYX_ERR(1, 796, __pyx_L1_error) } #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_3 = PyTuple_GET_ITEM(sequence, 0); __pyx_t_4 = PyTuple_GET_ITEM(sequence, 1); __Pyx_INCREF(__pyx_t_3); __Pyx_INCREF(__pyx_t_4); #else __pyx_t_3 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 796, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 796, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); #endif } else { __Pyx_RaiseNoneNotIterableError(); __PYX_ERR(1, 796, __pyx_L1_error) } if (!(likely(((__pyx_t_3) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_3, __pyx_ptype_5numpy_dtype))))) __PYX_ERR(1, 796, __pyx_L1_error) __Pyx_XDECREF_SET(__pyx_v_child, ((PyArray_Descr *)__pyx_t_3)); __pyx_t_3 = 0; __Pyx_XDECREF_SET(__pyx_v_new_offset, __pyx_t_4); __pyx_t_4 = 0; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":798 * child, new_offset = fields * * if (end - f) - (new_offset - offset[0]) < 15: # <<<<<<<<<<<<<< * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") * */ __pyx_t_4 = __Pyx_PyInt_From_int((__pyx_v_offset[0])); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 798, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_3 = PyNumber_Subtract(__pyx_v_new_offset, __pyx_t_4); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 798, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_5 = __Pyx_PyInt_As_int(__pyx_t_3); if (unlikely((__pyx_t_5 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 798, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_6 = ((((__pyx_v_end - __pyx_v_f) - ((int)__pyx_t_5)) < 15) != 0); if (__pyx_t_6) { /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":799 * * if (end - f) - (new_offset - offset[0]) < 15: * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") # <<<<<<<<<<<<<< * * if ((child.byteorder == c'>' and little_endian) or */ __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_RuntimeError, __pyx_tuple__8, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 799, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 799, __pyx_L1_error) /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":798 * child, new_offset = fields * * if (end - f) - (new_offset - offset[0]) < 15: # <<<<<<<<<<<<<< * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") * */ } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":801 * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") * * if ((child.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< * (child.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") */ __pyx_t_7 = ((__pyx_v_child->byteorder == '>') != 0); if (!__pyx_t_7) { goto __pyx_L8_next_or; } else { } __pyx_t_7 = (__pyx_v_little_endian != 0); if (!__pyx_t_7) { } else { __pyx_t_6 = __pyx_t_7; goto __pyx_L7_bool_binop_done; } __pyx_L8_next_or:; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":802 * * if ((child.byteorder == c'>' and little_endian) or * (child.byteorder == c'<' and not little_endian)): # <<<<<<<<<<<<<< * raise ValueError(u"Non-native byte order not supported") * # One could encode it in the format string and have Cython */ __pyx_t_7 = ((__pyx_v_child->byteorder == '<') != 0); if (__pyx_t_7) { } else { __pyx_t_6 = __pyx_t_7; goto __pyx_L7_bool_binop_done; } __pyx_t_7 = ((!(__pyx_v_little_endian != 0)) != 0); __pyx_t_6 = __pyx_t_7; __pyx_L7_bool_binop_done:; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":801 * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") * * if ((child.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< * (child.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") */ if (__pyx_t_6) { /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":803 * if ((child.byteorder == c'>' and little_endian) or * (child.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") # <<<<<<<<<<<<<< * # One could encode it in the format string and have Cython * # complain instead, BUT: < and > in format strings also imply */ __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__9, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 803, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 803, __pyx_L1_error) /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":801 * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") * * if ((child.byteorder == c'>' and little_endian) or # <<<<<<<<<<<<<< * (child.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") */ } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":813 * * # Output padding bytes * while offset[0] < new_offset: # <<<<<<<<<<<<<< * f[0] = 120 # "x"; pad byte * f += 1 */ while (1) { __pyx_t_3 = __Pyx_PyInt_From_int((__pyx_v_offset[0])); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 813, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = PyObject_RichCompare(__pyx_t_3, __pyx_v_new_offset, Py_LT); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 813, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 813, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (!__pyx_t_6) break; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":814 * # Output padding bytes * while offset[0] < new_offset: * f[0] = 120 # "x"; pad byte # <<<<<<<<<<<<<< * f += 1 * offset[0] += 1 */ (__pyx_v_f[0]) = 0x78; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":815 * while offset[0] < new_offset: * f[0] = 120 # "x"; pad byte * f += 1 # <<<<<<<<<<<<<< * offset[0] += 1 * */ __pyx_v_f = (__pyx_v_f + 1); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":816 * f[0] = 120 # "x"; pad byte * f += 1 * offset[0] += 1 # <<<<<<<<<<<<<< * * offset[0] += child.itemsize */ __pyx_t_8 = 0; (__pyx_v_offset[__pyx_t_8]) = ((__pyx_v_offset[__pyx_t_8]) + 1); } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":818 * offset[0] += 1 * * offset[0] += child.itemsize # <<<<<<<<<<<<<< * * if not PyDataType_HASFIELDS(child): */ __pyx_t_8 = 0; (__pyx_v_offset[__pyx_t_8]) = ((__pyx_v_offset[__pyx_t_8]) + __pyx_v_child->elsize); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":820 * offset[0] += child.itemsize * * if not PyDataType_HASFIELDS(child): # <<<<<<<<<<<<<< * t = child.type_num * if end - f < 5: */ __pyx_t_6 = ((!(PyDataType_HASFIELDS(__pyx_v_child) != 0)) != 0); if (__pyx_t_6) { /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":821 * * if not PyDataType_HASFIELDS(child): * t = child.type_num # <<<<<<<<<<<<<< * if end - f < 5: * raise RuntimeError(u"Format string allocated too short.") */ __pyx_t_4 = __Pyx_PyInt_From_int(__pyx_v_child->type_num); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 821, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_XDECREF_SET(__pyx_v_t, __pyx_t_4); __pyx_t_4 = 0; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":822 * if not PyDataType_HASFIELDS(child): * t = child.type_num * if end - f < 5: # <<<<<<<<<<<<<< * raise RuntimeError(u"Format string allocated too short.") * */ __pyx_t_6 = (((__pyx_v_end - __pyx_v_f) < 5) != 0); if (__pyx_t_6) { /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":823 * t = child.type_num * if end - f < 5: * raise RuntimeError(u"Format string allocated too short.") # <<<<<<<<<<<<<< * * # Until ticket #99 is fixed, use integers to avoid warnings */ __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_RuntimeError, __pyx_tuple__10, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 823, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_Raise(__pyx_t_4, 0, 0, 0); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __PYX_ERR(1, 823, __pyx_L1_error) /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":822 * if not PyDataType_HASFIELDS(child): * t = child.type_num * if end - f < 5: # <<<<<<<<<<<<<< * raise RuntimeError(u"Format string allocated too short.") * */ } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":826 * * # Until ticket #99 is fixed, use integers to avoid warnings * if t == NPY_BYTE: f[0] = 98 #"b" # <<<<<<<<<<<<<< * elif t == NPY_UBYTE: f[0] = 66 #"B" * elif t == NPY_SHORT: f[0] = 104 #"h" */ __pyx_t_4 = __Pyx_PyInt_From_enum__NPY_TYPES(NPY_BYTE); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 826, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_3 = PyObject_RichCompare(__pyx_v_t, __pyx_t_4, Py_EQ); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 826, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 826, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; if (__pyx_t_6) { (__pyx_v_f[0]) = 98; goto __pyx_L15; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":827 * # Until ticket #99 is fixed, use integers to avoid warnings * if t == NPY_BYTE: f[0] = 98 #"b" * elif t == NPY_UBYTE: f[0] = 66 #"B" # <<<<<<<<<<<<<< * elif t == NPY_SHORT: f[0] = 104 #"h" * elif t == NPY_USHORT: f[0] = 72 #"H" */ __pyx_t_3 = __Pyx_PyInt_From_enum__NPY_TYPES(NPY_UBYTE); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 827, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = PyObject_RichCompare(__pyx_v_t, __pyx_t_3, Py_EQ); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 827, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 827, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (__pyx_t_6) { (__pyx_v_f[0]) = 66; goto __pyx_L15; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":828 * if t == NPY_BYTE: f[0] = 98 #"b" * elif t == NPY_UBYTE: f[0] = 66 #"B" * elif t == NPY_SHORT: f[0] = 104 #"h" # <<<<<<<<<<<<<< * elif t == NPY_USHORT: f[0] = 72 #"H" * elif t == NPY_INT: f[0] = 105 #"i" */ __pyx_t_4 = __Pyx_PyInt_From_enum__NPY_TYPES(NPY_SHORT); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 828, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_3 = PyObject_RichCompare(__pyx_v_t, __pyx_t_4, Py_EQ); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 828, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 828, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; if (__pyx_t_6) { (__pyx_v_f[0]) = 0x68; goto __pyx_L15; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":829 * elif t == NPY_UBYTE: f[0] = 66 #"B" * elif t == NPY_SHORT: f[0] = 104 #"h" * elif t == NPY_USHORT: f[0] = 72 #"H" # <<<<<<<<<<<<<< * elif t == NPY_INT: f[0] = 105 #"i" * elif t == NPY_UINT: f[0] = 73 #"I" */ __pyx_t_3 = __Pyx_PyInt_From_enum__NPY_TYPES(NPY_USHORT); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 829, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = PyObject_RichCompare(__pyx_v_t, __pyx_t_3, Py_EQ); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 829, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 829, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (__pyx_t_6) { (__pyx_v_f[0]) = 72; goto __pyx_L15; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":830 * elif t == NPY_SHORT: f[0] = 104 #"h" * elif t == NPY_USHORT: f[0] = 72 #"H" * elif t == NPY_INT: f[0] = 105 #"i" # <<<<<<<<<<<<<< * elif t == NPY_UINT: f[0] = 73 #"I" * elif t == NPY_LONG: f[0] = 108 #"l" */ __pyx_t_4 = __Pyx_PyInt_From_enum__NPY_TYPES(NPY_INT); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 830, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_3 = PyObject_RichCompare(__pyx_v_t, __pyx_t_4, Py_EQ); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 830, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 830, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; if (__pyx_t_6) { (__pyx_v_f[0]) = 0x69; goto __pyx_L15; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":831 * elif t == NPY_USHORT: f[0] = 72 #"H" * elif t == NPY_INT: f[0] = 105 #"i" * elif t == NPY_UINT: f[0] = 73 #"I" # <<<<<<<<<<<<<< * elif t == NPY_LONG: f[0] = 108 #"l" * elif t == NPY_ULONG: f[0] = 76 #"L" */ __pyx_t_3 = __Pyx_PyInt_From_enum__NPY_TYPES(NPY_UINT); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 831, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = PyObject_RichCompare(__pyx_v_t, __pyx_t_3, Py_EQ); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 831, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 831, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (__pyx_t_6) { (__pyx_v_f[0]) = 73; goto __pyx_L15; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":832 * elif t == NPY_INT: f[0] = 105 #"i" * elif t == NPY_UINT: f[0] = 73 #"I" * elif t == NPY_LONG: f[0] = 108 #"l" # <<<<<<<<<<<<<< * elif t == NPY_ULONG: f[0] = 76 #"L" * elif t == NPY_LONGLONG: f[0] = 113 #"q" */ __pyx_t_4 = __Pyx_PyInt_From_enum__NPY_TYPES(NPY_LONG); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 832, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_3 = PyObject_RichCompare(__pyx_v_t, __pyx_t_4, Py_EQ); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 832, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 832, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; if (__pyx_t_6) { (__pyx_v_f[0]) = 0x6C; goto __pyx_L15; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":833 * elif t == NPY_UINT: f[0] = 73 #"I" * elif t == NPY_LONG: f[0] = 108 #"l" * elif t == NPY_ULONG: f[0] = 76 #"L" # <<<<<<<<<<<<<< * elif t == NPY_LONGLONG: f[0] = 113 #"q" * elif t == NPY_ULONGLONG: f[0] = 81 #"Q" */ __pyx_t_3 = __Pyx_PyInt_From_enum__NPY_TYPES(NPY_ULONG); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 833, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = PyObject_RichCompare(__pyx_v_t, __pyx_t_3, Py_EQ); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 833, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 833, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (__pyx_t_6) { (__pyx_v_f[0]) = 76; goto __pyx_L15; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":834 * elif t == NPY_LONG: f[0] = 108 #"l" * elif t == NPY_ULONG: f[0] = 76 #"L" * elif t == NPY_LONGLONG: f[0] = 113 #"q" # <<<<<<<<<<<<<< * elif t == NPY_ULONGLONG: f[0] = 81 #"Q" * elif t == NPY_FLOAT: f[0] = 102 #"f" */ __pyx_t_4 = __Pyx_PyInt_From_enum__NPY_TYPES(NPY_LONGLONG); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 834, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_3 = PyObject_RichCompare(__pyx_v_t, __pyx_t_4, Py_EQ); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 834, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 834, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; if (__pyx_t_6) { (__pyx_v_f[0]) = 0x71; goto __pyx_L15; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":835 * elif t == NPY_ULONG: f[0] = 76 #"L" * elif t == NPY_LONGLONG: f[0] = 113 #"q" * elif t == NPY_ULONGLONG: f[0] = 81 #"Q" # <<<<<<<<<<<<<< * elif t == NPY_FLOAT: f[0] = 102 #"f" * elif t == NPY_DOUBLE: f[0] = 100 #"d" */ __pyx_t_3 = __Pyx_PyInt_From_enum__NPY_TYPES(NPY_ULONGLONG); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 835, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = PyObject_RichCompare(__pyx_v_t, __pyx_t_3, Py_EQ); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 835, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 835, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (__pyx_t_6) { (__pyx_v_f[0]) = 81; goto __pyx_L15; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":836 * elif t == NPY_LONGLONG: f[0] = 113 #"q" * elif t == NPY_ULONGLONG: f[0] = 81 #"Q" * elif t == NPY_FLOAT: f[0] = 102 #"f" # <<<<<<<<<<<<<< * elif t == NPY_DOUBLE: f[0] = 100 #"d" * elif t == NPY_LONGDOUBLE: f[0] = 103 #"g" */ __pyx_t_4 = __Pyx_PyInt_From_enum__NPY_TYPES(NPY_FLOAT); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 836, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_3 = PyObject_RichCompare(__pyx_v_t, __pyx_t_4, Py_EQ); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 836, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 836, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; if (__pyx_t_6) { (__pyx_v_f[0]) = 0x66; goto __pyx_L15; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":837 * elif t == NPY_ULONGLONG: f[0] = 81 #"Q" * elif t == NPY_FLOAT: f[0] = 102 #"f" * elif t == NPY_DOUBLE: f[0] = 100 #"d" # <<<<<<<<<<<<<< * elif t == NPY_LONGDOUBLE: f[0] = 103 #"g" * elif t == NPY_CFLOAT: f[0] = 90; f[1] = 102; f += 1 # Zf */ __pyx_t_3 = __Pyx_PyInt_From_enum__NPY_TYPES(NPY_DOUBLE); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 837, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = PyObject_RichCompare(__pyx_v_t, __pyx_t_3, Py_EQ); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 837, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 837, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (__pyx_t_6) { (__pyx_v_f[0]) = 0x64; goto __pyx_L15; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":838 * elif t == NPY_FLOAT: f[0] = 102 #"f" * elif t == NPY_DOUBLE: f[0] = 100 #"d" * elif t == NPY_LONGDOUBLE: f[0] = 103 #"g" # <<<<<<<<<<<<<< * elif t == NPY_CFLOAT: f[0] = 90; f[1] = 102; f += 1 # Zf * elif t == NPY_CDOUBLE: f[0] = 90; f[1] = 100; f += 1 # Zd */ __pyx_t_4 = __Pyx_PyInt_From_enum__NPY_TYPES(NPY_LONGDOUBLE); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 838, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_3 = PyObject_RichCompare(__pyx_v_t, __pyx_t_4, Py_EQ); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 838, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 838, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; if (__pyx_t_6) { (__pyx_v_f[0]) = 0x67; goto __pyx_L15; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":839 * elif t == NPY_DOUBLE: f[0] = 100 #"d" * elif t == NPY_LONGDOUBLE: f[0] = 103 #"g" * elif t == NPY_CFLOAT: f[0] = 90; f[1] = 102; f += 1 # Zf # <<<<<<<<<<<<<< * elif t == NPY_CDOUBLE: f[0] = 90; f[1] = 100; f += 1 # Zd * elif t == NPY_CLONGDOUBLE: f[0] = 90; f[1] = 103; f += 1 # Zg */ __pyx_t_3 = __Pyx_PyInt_From_enum__NPY_TYPES(NPY_CFLOAT); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 839, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = PyObject_RichCompare(__pyx_v_t, __pyx_t_3, Py_EQ); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 839, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 839, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (__pyx_t_6) { (__pyx_v_f[0]) = 90; (__pyx_v_f[1]) = 0x66; __pyx_v_f = (__pyx_v_f + 1); goto __pyx_L15; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":840 * elif t == NPY_LONGDOUBLE: f[0] = 103 #"g" * elif t == NPY_CFLOAT: f[0] = 90; f[1] = 102; f += 1 # Zf * elif t == NPY_CDOUBLE: f[0] = 90; f[1] = 100; f += 1 # Zd # <<<<<<<<<<<<<< * elif t == NPY_CLONGDOUBLE: f[0] = 90; f[1] = 103; f += 1 # Zg * elif t == NPY_OBJECT: f[0] = 79 #"O" */ __pyx_t_4 = __Pyx_PyInt_From_enum__NPY_TYPES(NPY_CDOUBLE); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 840, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_3 = PyObject_RichCompare(__pyx_v_t, __pyx_t_4, Py_EQ); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 840, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 840, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; if (__pyx_t_6) { (__pyx_v_f[0]) = 90; (__pyx_v_f[1]) = 0x64; __pyx_v_f = (__pyx_v_f + 1); goto __pyx_L15; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":841 * elif t == NPY_CFLOAT: f[0] = 90; f[1] = 102; f += 1 # Zf * elif t == NPY_CDOUBLE: f[0] = 90; f[1] = 100; f += 1 # Zd * elif t == NPY_CLONGDOUBLE: f[0] = 90; f[1] = 103; f += 1 # Zg # <<<<<<<<<<<<<< * elif t == NPY_OBJECT: f[0] = 79 #"O" * else: */ __pyx_t_3 = __Pyx_PyInt_From_enum__NPY_TYPES(NPY_CLONGDOUBLE); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 841, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = PyObject_RichCompare(__pyx_v_t, __pyx_t_3, Py_EQ); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 841, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 841, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (__pyx_t_6) { (__pyx_v_f[0]) = 90; (__pyx_v_f[1]) = 0x67; __pyx_v_f = (__pyx_v_f + 1); goto __pyx_L15; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":842 * elif t == NPY_CDOUBLE: f[0] = 90; f[1] = 100; f += 1 # Zd * elif t == NPY_CLONGDOUBLE: f[0] = 90; f[1] = 103; f += 1 # Zg * elif t == NPY_OBJECT: f[0] = 79 #"O" # <<<<<<<<<<<<<< * else: * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) */ __pyx_t_4 = __Pyx_PyInt_From_enum__NPY_TYPES(NPY_OBJECT); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 842, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_3 = PyObject_RichCompare(__pyx_v_t, __pyx_t_4, Py_EQ); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 842, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 842, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; if (__pyx_t_6) { (__pyx_v_f[0]) = 79; goto __pyx_L15; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":844 * elif t == NPY_OBJECT: f[0] = 79 #"O" * else: * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) # <<<<<<<<<<<<<< * f += 1 * else: */ /*else*/ { __pyx_t_3 = PyUnicode_Format(__pyx_kp_u_unknown_dtype_code_in_numpy_pxd, __pyx_v_t); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 844, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 844, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_t_4, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 844, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __PYX_ERR(1, 844, __pyx_L1_error) } __pyx_L15:; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":845 * else: * raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) * f += 1 # <<<<<<<<<<<<<< * else: * # Cython ignores struct boundary information ("T{...}"), */ __pyx_v_f = (__pyx_v_f + 1); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":820 * offset[0] += child.itemsize * * if not PyDataType_HASFIELDS(child): # <<<<<<<<<<<<<< * t = child.type_num * if end - f < 5: */ goto __pyx_L13; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":849 * # Cython ignores struct boundary information ("T{...}"), * # so don't output it * f = _util_dtypestring(child, f, end, offset) # <<<<<<<<<<<<<< * return f * */ /*else*/ { __pyx_t_9 = __pyx_f_5numpy__util_dtypestring(__pyx_v_child, __pyx_v_f, __pyx_v_end, __pyx_v_offset); if (unlikely(__pyx_t_9 == NULL)) __PYX_ERR(1, 849, __pyx_L1_error) __pyx_v_f = __pyx_t_9; } __pyx_L13:; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":794 * cdef tuple fields * * for childname in descr.names: # <<<<<<<<<<<<<< * fields = descr.fields[childname] * child, new_offset = fields */ } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":850 * # so don't output it * f = _util_dtypestring(child, f, end, offset) * return f # <<<<<<<<<<<<<< * * */ __pyx_r = __pyx_v_f; goto __pyx_L0; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":785 * return PyArray_MultiIterNew(5, a, b, c, d, e) * * cdef inline char* _util_dtypestring(dtype descr, char* f, char* end, int* offset) except NULL: # <<<<<<<<<<<<<< * # Recursive utility function used in __getbuffer__ to get format * # string. The new location in the format string is returned. */ /* function exit code */ __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); __Pyx_XDECREF(__pyx_t_3); __Pyx_XDECREF(__pyx_t_4); __Pyx_AddTraceback("numpy._util_dtypestring", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; __Pyx_XDECREF((PyObject *)__pyx_v_child); __Pyx_XDECREF(__pyx_v_fields); __Pyx_XDECREF(__pyx_v_childname); __Pyx_XDECREF(__pyx_v_new_offset); __Pyx_XDECREF(__pyx_v_t); __Pyx_RefNannyFinishContext(); return __pyx_r; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":966 * * * cdef inline void set_array_base(ndarray arr, object base): # <<<<<<<<<<<<<< * cdef PyObject* baseptr * if base is None: */ static CYTHON_INLINE void __pyx_f_5numpy_set_array_base(PyArrayObject *__pyx_v_arr, PyObject *__pyx_v_base) { PyObject *__pyx_v_baseptr; __Pyx_RefNannyDeclarations int __pyx_t_1; int __pyx_t_2; __Pyx_RefNannySetupContext("set_array_base", 0); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":968 * cdef inline void set_array_base(ndarray arr, object base): * cdef PyObject* baseptr * if base is None: # <<<<<<<<<<<<<< * baseptr = NULL * else: */ __pyx_t_1 = (__pyx_v_base == Py_None); __pyx_t_2 = (__pyx_t_1 != 0); if (__pyx_t_2) { /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":969 * cdef PyObject* baseptr * if base is None: * baseptr = NULL # <<<<<<<<<<<<<< * else: * Py_INCREF(base) # important to do this before decref below! */ __pyx_v_baseptr = NULL; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":968 * cdef inline void set_array_base(ndarray arr, object base): * cdef PyObject* baseptr * if base is None: # <<<<<<<<<<<<<< * baseptr = NULL * else: */ goto __pyx_L3; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":971 * baseptr = NULL * else: * Py_INCREF(base) # important to do this before decref below! # <<<<<<<<<<<<<< * baseptr = base * Py_XDECREF(arr.base) */ /*else*/ { Py_INCREF(__pyx_v_base); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":972 * else: * Py_INCREF(base) # important to do this before decref below! * baseptr = base # <<<<<<<<<<<<<< * Py_XDECREF(arr.base) * arr.base = baseptr */ __pyx_v_baseptr = ((PyObject *)__pyx_v_base); } __pyx_L3:; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":973 * Py_INCREF(base) # important to do this before decref below! * baseptr = base * Py_XDECREF(arr.base) # <<<<<<<<<<<<<< * arr.base = baseptr * */ Py_XDECREF(__pyx_v_arr->base); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":974 * baseptr = base * Py_XDECREF(arr.base) * arr.base = baseptr # <<<<<<<<<<<<<< * * cdef inline object get_array_base(ndarray arr): */ __pyx_v_arr->base = __pyx_v_baseptr; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":966 * * * cdef inline void set_array_base(ndarray arr, object base): # <<<<<<<<<<<<<< * cdef PyObject* baseptr * if base is None: */ /* function exit code */ __Pyx_RefNannyFinishContext(); } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":976 * arr.base = baseptr * * cdef inline object get_array_base(ndarray arr): # <<<<<<<<<<<<<< * if arr.base is NULL: * return None */ static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__pyx_v_arr) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations int __pyx_t_1; __Pyx_RefNannySetupContext("get_array_base", 0); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":977 * * cdef inline object get_array_base(ndarray arr): * if arr.base is NULL: # <<<<<<<<<<<<<< * return None * else: */ __pyx_t_1 = ((__pyx_v_arr->base == NULL) != 0); if (__pyx_t_1) { /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":978 * cdef inline object get_array_base(ndarray arr): * if arr.base is NULL: * return None # <<<<<<<<<<<<<< * else: * return arr.base */ __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(Py_None); __pyx_r = Py_None; goto __pyx_L0; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":977 * * cdef inline object get_array_base(ndarray arr): * if arr.base is NULL: # <<<<<<<<<<<<<< * return None * else: */ } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":980 * return None * else: * return arr.base # <<<<<<<<<<<<<< */ /*else*/ { __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(((PyObject *)__pyx_v_arr->base)); __pyx_r = ((PyObject *)__pyx_v_arr->base); goto __pyx_L0; } /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":976 * arr.base = baseptr * * cdef inline object get_array_base(ndarray arr): # <<<<<<<<<<<<<< * if arr.base is NULL: * return None */ /* function exit code */ __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); return __pyx_r; } static PyMethodDef __pyx_methods[] = { {0, 0, 0, 0} }; #if PY_MAJOR_VERSION >= 3 static struct PyModuleDef __pyx_moduledef = { #if PY_VERSION_HEX < 0x03020000 { PyObject_HEAD_INIT(NULL) NULL, 0, NULL }, #else PyModuleDef_HEAD_INIT, #endif "gpu_nms", 0, /* m_doc */ -1, /* m_size */ __pyx_methods /* m_methods */, NULL, /* m_reload */ NULL, /* m_traverse */ NULL, /* m_clear */ NULL /* m_free */ }; #endif static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp_s_D_v_zix_caffe_caffe_win_20160523, __pyx_k_D_v_zix_caffe_caffe_win_20160523, sizeof(__pyx_k_D_v_zix_caffe_caffe_win_20160523), 0, 0, 1, 0}, {&__pyx_kp_u_Format_string_allocated_too_shor, __pyx_k_Format_string_allocated_too_shor, sizeof(__pyx_k_Format_string_allocated_too_shor), 0, 1, 0, 0}, {&__pyx_kp_u_Format_string_allocated_too_shor_2, __pyx_k_Format_string_allocated_too_shor_2, sizeof(__pyx_k_Format_string_allocated_too_shor_2), 0, 1, 0, 0}, {&__pyx_kp_u_Non_native_byte_order_not_suppor, __pyx_k_Non_native_byte_order_not_suppor, sizeof(__pyx_k_Non_native_byte_order_not_suppor), 0, 1, 0, 0}, {&__pyx_n_s_RuntimeError, __pyx_k_RuntimeError, sizeof(__pyx_k_RuntimeError), 0, 0, 1, 1}, {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1}, {&__pyx_n_s_argsort, __pyx_k_argsort, sizeof(__pyx_k_argsort), 0, 0, 1, 1}, {&__pyx_n_s_boxes_dim, __pyx_k_boxes_dim, sizeof(__pyx_k_boxes_dim), 0, 0, 1, 1}, {&__pyx_n_s_boxes_num, __pyx_k_boxes_num, sizeof(__pyx_k_boxes_num), 0, 0, 1, 1}, {&__pyx_n_s_dets, __pyx_k_dets, sizeof(__pyx_k_dets), 0, 0, 1, 1}, {&__pyx_n_s_device_id, __pyx_k_device_id, sizeof(__pyx_k_device_id), 0, 0, 1, 1}, {&__pyx_n_s_dtype, __pyx_k_dtype, sizeof(__pyx_k_dtype), 0, 0, 1, 1}, {&__pyx_n_s_gpu_nms, __pyx_k_gpu_nms, sizeof(__pyx_k_gpu_nms), 0, 0, 1, 1}, {&__pyx_n_s_import, __pyx_k_import, sizeof(__pyx_k_import), 0, 0, 1, 1}, {&__pyx_n_s_int32, __pyx_k_int32, sizeof(__pyx_k_int32), 0, 0, 1, 1}, {&__pyx_n_s_keep, __pyx_k_keep, sizeof(__pyx_k_keep), 0, 0, 1, 1}, {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, {&__pyx_kp_u_ndarray_is_not_C_contiguous, __pyx_k_ndarray_is_not_C_contiguous, sizeof(__pyx_k_ndarray_is_not_C_contiguous), 0, 1, 0, 0}, {&__pyx_kp_u_ndarray_is_not_Fortran_contiguou, __pyx_k_ndarray_is_not_Fortran_contiguou, sizeof(__pyx_k_ndarray_is_not_Fortran_contiguou), 0, 1, 0, 0}, {&__pyx_n_s_nms_gpu_nms, __pyx_k_nms_gpu_nms, sizeof(__pyx_k_nms_gpu_nms), 0, 0, 1, 1}, {&__pyx_n_s_np, __pyx_k_np, sizeof(__pyx_k_np), 0, 0, 1, 1}, {&__pyx_n_s_num_out, __pyx_k_num_out, sizeof(__pyx_k_num_out), 0, 0, 1, 1}, {&__pyx_n_s_numpy, __pyx_k_numpy, sizeof(__pyx_k_numpy), 0, 0, 1, 1}, {&__pyx_n_s_order, __pyx_k_order, sizeof(__pyx_k_order), 0, 0, 1, 1}, {&__pyx_n_s_range, __pyx_k_range, sizeof(__pyx_k_range), 0, 0, 1, 1}, {&__pyx_n_s_scores, __pyx_k_scores, sizeof(__pyx_k_scores), 0, 0, 1, 1}, {&__pyx_n_s_sorted_dets, __pyx_k_sorted_dets, sizeof(__pyx_k_sorted_dets), 0, 0, 1, 1}, {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, {&__pyx_n_s_thresh, __pyx_k_thresh, sizeof(__pyx_k_thresh), 0, 0, 1, 1}, {&__pyx_kp_u_unknown_dtype_code_in_numpy_pxd, __pyx_k_unknown_dtype_code_in_numpy_pxd, sizeof(__pyx_k_unknown_dtype_code_in_numpy_pxd), 0, 1, 0, 0}, {&__pyx_n_s_zeros, __pyx_k_zeros, sizeof(__pyx_k_zeros), 0, 0, 1, 1}, {0, 0, 0, 0, 0, 0, 0} }; static int __Pyx_InitCachedBuiltins(void) { __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(1, 218, __pyx_L1_error) __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(1, 231, __pyx_L1_error) __pyx_builtin_RuntimeError = __Pyx_GetBuiltinName(__pyx_n_s_RuntimeError); if (!__pyx_builtin_RuntimeError) __PYX_ERR(1, 799, __pyx_L1_error) return 0; __pyx_L1_error:; return -1; } static int __Pyx_InitCachedConstants(void) { __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); /* "nms/gpu_nms.pyx":24 * keep = np.zeros(boxes_num, dtype=np.int32) * cdef np.ndarray[np.float32_t, ndim=1] \ * scores = dets[:, 4] # <<<<<<<<<<<<<< * #cdef np.ndarray[np.int_t, ndim=1] \ // 20160601, by xzn * # order = scores.argsort()[::-1] */ __pyx_slice_ = PySlice_New(Py_None, Py_None, Py_None); if (unlikely(!__pyx_slice_)) __PYX_ERR(0, 24, __pyx_L1_error) __Pyx_GOTREF(__pyx_slice_); __Pyx_GIVEREF(__pyx_slice_); __pyx_tuple__2 = PyTuple_Pack(2, __pyx_slice_, __pyx_int_4); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(0, 24, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__2); __Pyx_GIVEREF(__pyx_tuple__2); /* "nms/gpu_nms.pyx":28 * # order = scores.argsort()[::-1] * cdef np.ndarray[np.intp_t, ndim=1] \ * order = scores.argsort()[::-1] # <<<<<<<<<<<<<< * cdef np.ndarray[np.float32_t, ndim=2] \ * sorted_dets = dets[order, :] */ __pyx_slice__3 = PySlice_New(Py_None, Py_None, __pyx_int_neg_1); if (unlikely(!__pyx_slice__3)) __PYX_ERR(0, 28, __pyx_L1_error) __Pyx_GOTREF(__pyx_slice__3); __Pyx_GIVEREF(__pyx_slice__3); /* "nms/gpu_nms.pyx":30 * order = scores.argsort()[::-1] * cdef np.ndarray[np.float32_t, ndim=2] \ * sorted_dets = dets[order, :] # <<<<<<<<<<<<<< * _nms(&keep[0], &num_out, &sorted_dets[0, 0], boxes_num, boxes_dim, thresh, device_id) * keep = keep[:num_out] */ __pyx_slice__4 = PySlice_New(Py_None, Py_None, Py_None); if (unlikely(!__pyx_slice__4)) __PYX_ERR(0, 30, __pyx_L1_error) __Pyx_GOTREF(__pyx_slice__4); __Pyx_GIVEREF(__pyx_slice__4); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":218 * if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_C_CONTIGUOUS)): * raise ValueError(u"ndarray is not C contiguous") # <<<<<<<<<<<<<< * * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) */ __pyx_tuple__5 = PyTuple_Pack(1, __pyx_kp_u_ndarray_is_not_C_contiguous); if (unlikely(!__pyx_tuple__5)) __PYX_ERR(1, 218, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__5); __Pyx_GIVEREF(__pyx_tuple__5); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":222 * if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) * and not PyArray_CHKFLAGS(self, NPY_F_CONTIGUOUS)): * raise ValueError(u"ndarray is not Fortran contiguous") # <<<<<<<<<<<<<< * * info.buf = PyArray_DATA(self) */ __pyx_tuple__6 = PyTuple_Pack(1, __pyx_kp_u_ndarray_is_not_Fortran_contiguou); if (unlikely(!__pyx_tuple__6)) __PYX_ERR(1, 222, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__6); __Pyx_GIVEREF(__pyx_tuple__6); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":259 * if ((descr.byteorder == c'>' and little_endian) or * (descr.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") # <<<<<<<<<<<<<< * if t == NPY_BYTE: f = "b" * elif t == NPY_UBYTE: f = "B" */ __pyx_tuple__7 = PyTuple_Pack(1, __pyx_kp_u_Non_native_byte_order_not_suppor); if (unlikely(!__pyx_tuple__7)) __PYX_ERR(1, 259, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__7); __Pyx_GIVEREF(__pyx_tuple__7); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":799 * * if (end - f) - (new_offset - offset[0]) < 15: * raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") # <<<<<<<<<<<<<< * * if ((child.byteorder == c'>' and little_endian) or */ __pyx_tuple__8 = PyTuple_Pack(1, __pyx_kp_u_Format_string_allocated_too_shor); if (unlikely(!__pyx_tuple__8)) __PYX_ERR(1, 799, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__8); __Pyx_GIVEREF(__pyx_tuple__8); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":803 * if ((child.byteorder == c'>' and little_endian) or * (child.byteorder == c'<' and not little_endian)): * raise ValueError(u"Non-native byte order not supported") # <<<<<<<<<<<<<< * # One could encode it in the format string and have Cython * # complain instead, BUT: < and > in format strings also imply */ __pyx_tuple__9 = PyTuple_Pack(1, __pyx_kp_u_Non_native_byte_order_not_suppor); if (unlikely(!__pyx_tuple__9)) __PYX_ERR(1, 803, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__9); __Pyx_GIVEREF(__pyx_tuple__9); /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":823 * t = child.type_num * if end - f < 5: * raise RuntimeError(u"Format string allocated too short.") # <<<<<<<<<<<<<< * * # Until ticket #99 is fixed, use integers to avoid warnings */ __pyx_tuple__10 = PyTuple_Pack(1, __pyx_kp_u_Format_string_allocated_too_shor_2); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(1, 823, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__10); __Pyx_GIVEREF(__pyx_tuple__10); /* "nms/gpu_nms.pyx":16 * void _nms(np.int32_t*, int*, np.float32_t*, int, int, float, int) * * def gpu_nms(np.ndarray[np.float32_t, ndim=2] dets, np.float thresh, # <<<<<<<<<<<<<< * np.int32_t device_id=0): * cdef int boxes_num = dets.shape[0] */ __pyx_tuple__11 = PyTuple_Pack(10, __pyx_n_s_dets, __pyx_n_s_thresh, __pyx_n_s_device_id, __pyx_n_s_boxes_num, __pyx_n_s_boxes_dim, __pyx_n_s_num_out, __pyx_n_s_keep, __pyx_n_s_scores, __pyx_n_s_order, __pyx_n_s_sorted_dets); if (unlikely(!__pyx_tuple__11)) __PYX_ERR(0, 16, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__11); __Pyx_GIVEREF(__pyx_tuple__11); __pyx_codeobj__12 = (PyObject*)__Pyx_PyCode_New(3, 0, 10, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__11, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_D_v_zix_caffe_caffe_win_20160523, __pyx_n_s_gpu_nms, 16, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__12)) __PYX_ERR(0, 16, __pyx_L1_error) __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; __Pyx_RefNannyFinishContext(); return -1; } static int __Pyx_InitGlobals(void) { if (__Pyx_InitStrings(__pyx_string_tab) < 0) __PYX_ERR(0, 1, __pyx_L1_error); __pyx_int_4 = PyInt_FromLong(4); if (unlikely(!__pyx_int_4)) __PYX_ERR(0, 1, __pyx_L1_error) __pyx_int_neg_1 = PyInt_FromLong(-1); if (unlikely(!__pyx_int_neg_1)) __PYX_ERR(0, 1, __pyx_L1_error) return 0; __pyx_L1_error:; return -1; } #if PY_MAJOR_VERSION < 3 PyMODINIT_FUNC initgpu_nms(void); /*proto*/ PyMODINIT_FUNC initgpu_nms(void) #else PyMODINIT_FUNC PyInit_gpu_nms(void); /*proto*/ PyMODINIT_FUNC PyInit_gpu_nms(void) #endif { PyObject *__pyx_t_1 = NULL; __Pyx_RefNannyDeclarations #if CYTHON_REFNANNY __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); if (!__Pyx_RefNanny) { PyErr_Clear(); __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); if (!__Pyx_RefNanny) Py_FatalError("failed to import 'refnanny' module"); } #endif __Pyx_RefNannySetupContext("PyMODINIT_FUNC PyInit_gpu_nms(void)", 0); if (__Pyx_check_binary_version() < 0) __PYX_ERR(0, 1, __pyx_L1_error) __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) __pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) __pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error) #ifdef __Pyx_CyFunction_USED if (__pyx_CyFunction_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) #endif #ifdef __Pyx_FusedFunction_USED if (__pyx_FusedFunction_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) #endif #ifdef __Pyx_Coroutine_USED if (__pyx_Coroutine_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) #endif #ifdef __Pyx_Generator_USED if (__pyx_Generator_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) #endif #ifdef __Pyx_StopAsyncIteration_USED if (__pyx_StopAsyncIteration_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) #endif /*--- Library function declarations ---*/ /*--- Threads initialization code ---*/ #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS #ifdef WITH_THREAD /* Python build with threading support? */ PyEval_InitThreads(); #endif #endif /*--- Module creation code ---*/ #if PY_MAJOR_VERSION < 3 __pyx_m = Py_InitModule4("gpu_nms", __pyx_methods, 0, 0, PYTHON_API_VERSION); Py_XINCREF(__pyx_m); #else __pyx_m = PyModule_Create(&__pyx_moduledef); #endif if (unlikely(!__pyx_m)) __PYX_ERR(0, 1, __pyx_L1_error) __pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) Py_INCREF(__pyx_d); __pyx_b = PyImport_AddModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) #if CYTHON_COMPILING_IN_PYPY Py_INCREF(__pyx_b); #endif if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error); /*--- Initialize various global constants etc. ---*/ if (__Pyx_InitGlobals() < 0) __PYX_ERR(0, 1, __pyx_L1_error) #if PY_MAJOR_VERSION < 3 && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) if (__Pyx_init_sys_getdefaultencoding_params() < 0) __PYX_ERR(0, 1, __pyx_L1_error) #endif if (__pyx_module_is_main_nms__gpu_nms) { if (PyObject_SetAttrString(__pyx_m, "__name__", __pyx_n_s_main) < 0) __PYX_ERR(0, 1, __pyx_L1_error) } #if PY_MAJOR_VERSION >= 3 { PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error) if (!PyDict_GetItemString(modules, "nms.gpu_nms")) { if (unlikely(PyDict_SetItemString(modules, "nms.gpu_nms", __pyx_m) < 0)) __PYX_ERR(0, 1, __pyx_L1_error) } } #endif /*--- Builtin init code ---*/ if (__Pyx_InitCachedBuiltins() < 0) __PYX_ERR(0, 1, __pyx_L1_error) /*--- Constants init code ---*/ if (__Pyx_InitCachedConstants() < 0) __PYX_ERR(0, 1, __pyx_L1_error) /*--- Global init code ---*/ /*--- Variable export code ---*/ /*--- Function export code ---*/ /*--- Type init code ---*/ /*--- Type import code ---*/ __pyx_ptype_7cpython_4type_type = __Pyx_ImportType(__Pyx_BUILTIN_MODULE_NAME, "type", #if CYTHON_COMPILING_IN_PYPY sizeof(PyTypeObject), #else sizeof(PyHeapTypeObject), #endif 0); if (unlikely(!__pyx_ptype_7cpython_4type_type)) __PYX_ERR(2, 9, __pyx_L1_error) __pyx_ptype_5numpy_dtype = __Pyx_ImportType("numpy", "dtype", sizeof(PyArray_Descr), 0); if (unlikely(!__pyx_ptype_5numpy_dtype)) __PYX_ERR(1, 155, __pyx_L1_error) __pyx_ptype_5numpy_flatiter = __Pyx_ImportType("numpy", "flatiter", sizeof(PyArrayIterObject), 0); if (unlikely(!__pyx_ptype_5numpy_flatiter)) __PYX_ERR(1, 168, __pyx_L1_error) __pyx_ptype_5numpy_broadcast = __Pyx_ImportType("numpy", "broadcast", sizeof(PyArrayMultiIterObject), 0); if (unlikely(!__pyx_ptype_5numpy_broadcast)) __PYX_ERR(1, 172, __pyx_L1_error) __pyx_ptype_5numpy_ndarray = __Pyx_ImportType("numpy", "ndarray", sizeof(PyArrayObject), 0); if (unlikely(!__pyx_ptype_5numpy_ndarray)) __PYX_ERR(1, 181, __pyx_L1_error) __pyx_ptype_5numpy_ufunc = __Pyx_ImportType("numpy", "ufunc", sizeof(PyUFuncObject), 0); if (unlikely(!__pyx_ptype_5numpy_ufunc)) __PYX_ERR(1, 861, __pyx_L1_error) /*--- Variable import code ---*/ /*--- Function import code ---*/ /*--- Execution code ---*/ #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) if (__Pyx_patch_abc() < 0) __PYX_ERR(0, 1, __pyx_L1_error) #endif /* "nms/gpu_nms.pyx":8 * # -------------------------------------------------------- * * import numpy as np # <<<<<<<<<<<<<< * cimport numpy as np * */ __pyx_t_1 = __Pyx_Import(__pyx_n_s_numpy, 0, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 8, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); if (PyDict_SetItem(__pyx_d, __pyx_n_s_np, __pyx_t_1) < 0) __PYX_ERR(0, 8, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; /* "nms/gpu_nms.pyx":11 * cimport numpy as np * * assert sizeof(int) == sizeof(np.int32_t) # <<<<<<<<<<<<<< * * cdef extern from "gpu_nms.hpp": */ #ifndef CYTHON_WITHOUT_ASSERTIONS if (unlikely(!Py_OptimizeFlag)) { if (unlikely(!(((sizeof(int)) == (sizeof(__pyx_t_5numpy_int32_t))) != 0))) { PyErr_SetNone(PyExc_AssertionError); __PYX_ERR(0, 11, __pyx_L1_error) } } #endif /* "nms/gpu_nms.pyx":16 * void _nms(np.int32_t*, int*, np.float32_t*, int, int, float, int) * * def gpu_nms(np.ndarray[np.float32_t, ndim=2] dets, np.float thresh, # <<<<<<<<<<<<<< * np.int32_t device_id=0): * cdef int boxes_num = dets.shape[0] */ __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_3nms_7gpu_nms_1gpu_nms, NULL, __pyx_n_s_nms_gpu_nms); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 16, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); if (PyDict_SetItem(__pyx_d, __pyx_n_s_gpu_nms, __pyx_t_1) < 0) __PYX_ERR(0, 16, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; /* "nms/gpu_nms.pyx":1 * # -------------------------------------------------------- # <<<<<<<<<<<<<< * # Faster R-CNN * # Copyright (c) 2015 Microsoft */ __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_1) < 0) __PYX_ERR(0, 1, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; /* "C:/Anaconda2/lib/site-packages/Cython/Includes/numpy/__init__.pxd":976 * arr.base = baseptr * * cdef inline object get_array_base(ndarray arr): # <<<<<<<<<<<<<< * if arr.base is NULL: * return None */ /*--- Wrapped vars code ---*/ goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); if (__pyx_m) { if (__pyx_d) { __Pyx_AddTraceback("init nms.gpu_nms", __pyx_clineno, __pyx_lineno, __pyx_filename); } Py_DECREF(__pyx_m); __pyx_m = 0; } else if (!PyErr_Occurred()) { PyErr_SetString(PyExc_ImportError, "init nms.gpu_nms"); } __pyx_L0:; __Pyx_RefNannyFinishContext(); #if PY_MAJOR_VERSION < 3 return; #else return __pyx_m; #endif } /* --- Runtime support code --- */ /* Refnanny */ #if CYTHON_REFNANNY static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { PyObject *m = NULL, *p = NULL; void *r = NULL; m = PyImport_ImportModule((char *)modname); if (!m) goto end; p = PyObject_GetAttrString(m, (char *)"RefNannyAPI"); if (!p) goto end; r = PyLong_AsVoidPtr(p); end: Py_XDECREF(p); Py_XDECREF(m); return (__Pyx_RefNannyAPIStruct *)r; } #endif /* RaiseArgTupleInvalid */ static void __Pyx_RaiseArgtupleInvalid( const char* func_name, int exact, Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found) { Py_ssize_t num_expected; const char *more_or_less; if (num_found < num_min) { num_expected = num_min; more_or_less = "at least"; } else { num_expected = num_max; more_or_less = "at most"; } if (exact) { more_or_less = "exactly"; } PyErr_Format(PyExc_TypeError, "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", func_name, more_or_less, num_expected, (num_expected == 1) ? "" : "s", num_found); } /* RaiseDoubleKeywords */ static void __Pyx_RaiseDoubleKeywordsError( const char* func_name, PyObject* kw_name) { PyErr_Format(PyExc_TypeError, #if PY_MAJOR_VERSION >= 3 "%s() got multiple values for keyword argument '%U'", func_name, kw_name); #else "%s() got multiple values for keyword argument '%s'", func_name, PyString_AsString(kw_name)); #endif } /* ParseKeywords */ static int __Pyx_ParseOptionalKeywords( PyObject *kwds, PyObject **argnames[], PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, const char* function_name) { PyObject *key = 0, *value = 0; Py_ssize_t pos = 0; PyObject*** name; PyObject*** first_kw_arg = argnames + num_pos_args; while (PyDict_Next(kwds, &pos, &key, &value)) { name = first_kw_arg; while (*name && (**name != key)) name++; if (*name) { values[name-argnames] = value; continue; } name = first_kw_arg; #if PY_MAJOR_VERSION < 3 if (likely(PyString_CheckExact(key)) || likely(PyString_Check(key))) { while (*name) { if ((CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**name) == PyString_GET_SIZE(key)) && _PyString_Eq(**name, key)) { values[name-argnames] = value; break; } name++; } if (*name) continue; else { PyObject*** argname = argnames; while (argname != first_kw_arg) { if ((**argname == key) || ( (CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**argname) == PyString_GET_SIZE(key)) && _PyString_Eq(**argname, key))) { goto arg_passed_twice; } argname++; } } } else #endif if (likely(PyUnicode_Check(key))) { while (*name) { int cmp = (**name == key) ? 0 : #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 (PyUnicode_GET_SIZE(**name) != PyUnicode_GET_SIZE(key)) ? 1 : #endif PyUnicode_Compare(**name, key); if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; if (cmp == 0) { values[name-argnames] = value; break; } name++; } if (*name) continue; else { PyObject*** argname = argnames; while (argname != first_kw_arg) { int cmp = (**argname == key) ? 0 : #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 (PyUnicode_GET_SIZE(**argname) != PyUnicode_GET_SIZE(key)) ? 1 : #endif PyUnicode_Compare(**argname, key); if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; if (cmp == 0) goto arg_passed_twice; argname++; } } } else goto invalid_keyword_type; if (kwds2) { if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; } else { goto invalid_keyword; } } return 0; arg_passed_twice: __Pyx_RaiseDoubleKeywordsError(function_name, key); goto bad; invalid_keyword_type: PyErr_Format(PyExc_TypeError, "%.200s() keywords must be strings", function_name); goto bad; invalid_keyword: PyErr_Format(PyExc_TypeError, #if PY_MAJOR_VERSION < 3 "%.200s() got an unexpected keyword argument '%.200s'", function_name, PyString_AsString(key)); #else "%s() got an unexpected keyword argument '%U'", function_name, key); #endif bad: return -1; } /* ArgTypeTest */ static void __Pyx_RaiseArgumentTypeInvalid(const char* name, PyObject *obj, PyTypeObject *type) { PyErr_Format(PyExc_TypeError, "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", name, type->tp_name, Py_TYPE(obj)->tp_name); } static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, const char *name, int exact) { if (unlikely(!type)) { PyErr_SetString(PyExc_SystemError, "Missing type object"); return 0; } if (none_allowed && obj == Py_None) return 1; else if (exact) { if (likely(Py_TYPE(obj) == type)) return 1; #if PY_MAJOR_VERSION == 2 else if ((type == &PyBaseString_Type) && likely(__Pyx_PyBaseString_CheckExact(obj))) return 1; #endif } else { if (likely(PyObject_TypeCheck(obj, type))) return 1; } __Pyx_RaiseArgumentTypeInvalid(name, obj, type); return 0; } /* BufferFormatCheck */ static CYTHON_INLINE int __Pyx_IsLittleEndian(void) { unsigned int n = 1; return *(unsigned char*)(&n) != 0; } static void __Pyx_BufFmt_Init(__Pyx_BufFmt_Context* ctx, __Pyx_BufFmt_StackElem* stack, __Pyx_TypeInfo* type) { stack[0].field = &ctx->root; stack[0].parent_offset = 0; ctx->root.type = type; ctx->root.name = "buffer dtype"; ctx->root.offset = 0; ctx->head = stack; ctx->head->field = &ctx->root; ctx->fmt_offset = 0; ctx->head->parent_offset = 0; ctx->new_packmode = '@'; ctx->enc_packmode = '@'; ctx->new_count = 1; ctx->enc_count = 0; ctx->enc_type = 0; ctx->is_complex = 0; ctx->is_valid_array = 0; ctx->struct_alignment = 0; while (type->typegroup == 'S') { ++ctx->head; ctx->head->field = type->fields; ctx->head->parent_offset = 0; type = type->fields->type; } } static int __Pyx_BufFmt_ParseNumber(const char** ts) { int count; const char* t = *ts; if (*t < '0' || *t > '9') { return -1; } else { count = *t++ - '0'; while (*t >= '0' && *t < '9') { count *= 10; count += *t++ - '0'; } } *ts = t; return count; } static int __Pyx_BufFmt_ExpectNumber(const char **ts) { int number = __Pyx_BufFmt_ParseNumber(ts); if (number == -1) PyErr_Format(PyExc_ValueError,\ "Does not understand character buffer dtype format string ('%c')", **ts); return number; } static void __Pyx_BufFmt_RaiseUnexpectedChar(char ch) { PyErr_Format(PyExc_ValueError, "Unexpected format string character: '%c'", ch); } static const char* __Pyx_BufFmt_DescribeTypeChar(char ch, int is_complex) { switch (ch) { case 'c': return "'char'"; case 'b': return "'signed char'"; case 'B': return "'unsigned char'"; case 'h': return "'short'"; case 'H': return "'unsigned short'"; case 'i': return "'int'"; case 'I': return "'unsigned int'"; case 'l': return "'long'"; case 'L': return "'unsigned long'"; case 'q': return "'long long'"; case 'Q': return "'unsigned long long'"; case 'f': return (is_complex ? "'complex float'" : "'float'"); case 'd': return (is_complex ? "'complex double'" : "'double'"); case 'g': return (is_complex ? "'complex long double'" : "'long double'"); case 'T': return "a struct"; case 'O': return "Python object"; case 'P': return "a pointer"; case 's': case 'p': return "a string"; case 0: return "end"; default: return "unparseable format string"; } } static size_t __Pyx_BufFmt_TypeCharToStandardSize(char ch, int is_complex) { switch (ch) { case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1; case 'h': case 'H': return 2; case 'i': case 'I': case 'l': case 'L': return 4; case 'q': case 'Q': return 8; case 'f': return (is_complex ? 8 : 4); case 'd': return (is_complex ? 16 : 8); case 'g': { PyErr_SetString(PyExc_ValueError, "Python does not define a standard format string size for long double ('g').."); return 0; } case 'O': case 'P': return sizeof(void*); default: __Pyx_BufFmt_RaiseUnexpectedChar(ch); return 0; } } static size_t __Pyx_BufFmt_TypeCharToNativeSize(char ch, int is_complex) { switch (ch) { case 'c': case 'b': case 'B': case 's': case 'p': return 1; case 'h': case 'H': return sizeof(short); case 'i': case 'I': return sizeof(int); case 'l': case 'L': return sizeof(long); #ifdef HAVE_LONG_LONG case 'q': case 'Q': return sizeof(PY_LONG_LONG); #endif case 'f': return sizeof(float) * (is_complex ? 2 : 1); case 'd': return sizeof(double) * (is_complex ? 2 : 1); case 'g': return sizeof(long double) * (is_complex ? 2 : 1); case 'O': case 'P': return sizeof(void*); default: { __Pyx_BufFmt_RaiseUnexpectedChar(ch); return 0; } } } typedef struct { char c; short x; } __Pyx_st_short; typedef struct { char c; int x; } __Pyx_st_int; typedef struct { char c; long x; } __Pyx_st_long; typedef struct { char c; float x; } __Pyx_st_float; typedef struct { char c; double x; } __Pyx_st_double; typedef struct { char c; long double x; } __Pyx_st_longdouble; typedef struct { char c; void *x; } __Pyx_st_void_p; #ifdef HAVE_LONG_LONG typedef struct { char c; PY_LONG_LONG x; } __Pyx_st_longlong; #endif static size_t __Pyx_BufFmt_TypeCharToAlignment(char ch, CYTHON_UNUSED int is_complex) { switch (ch) { case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1; case 'h': case 'H': return sizeof(__Pyx_st_short) - sizeof(short); case 'i': case 'I': return sizeof(__Pyx_st_int) - sizeof(int); case 'l': case 'L': return sizeof(__Pyx_st_long) - sizeof(long); #ifdef HAVE_LONG_LONG case 'q': case 'Q': return sizeof(__Pyx_st_longlong) - sizeof(PY_LONG_LONG); #endif case 'f': return sizeof(__Pyx_st_float) - sizeof(float); case 'd': return sizeof(__Pyx_st_double) - sizeof(double); case 'g': return sizeof(__Pyx_st_longdouble) - sizeof(long double); case 'P': case 'O': return sizeof(__Pyx_st_void_p) - sizeof(void*); default: __Pyx_BufFmt_RaiseUnexpectedChar(ch); return 0; } } /* These are for computing the padding at the end of the struct to align on the first member of the struct. This will probably the same as above, but we don't have any guarantees. */ typedef struct { short x; char c; } __Pyx_pad_short; typedef struct { int x; char c; } __Pyx_pad_int; typedef struct { long x; char c; } __Pyx_pad_long; typedef struct { float x; char c; } __Pyx_pad_float; typedef struct { double x; char c; } __Pyx_pad_double; typedef struct { long double x; char c; } __Pyx_pad_longdouble; typedef struct { void *x; char c; } __Pyx_pad_void_p; #ifdef HAVE_LONG_LONG typedef struct { PY_LONG_LONG x; char c; } __Pyx_pad_longlong; #endif static size_t __Pyx_BufFmt_TypeCharToPadding(char ch, CYTHON_UNUSED int is_complex) { switch (ch) { case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1; case 'h': case 'H': return sizeof(__Pyx_pad_short) - sizeof(short); case 'i': case 'I': return sizeof(__Pyx_pad_int) - sizeof(int); case 'l': case 'L': return sizeof(__Pyx_pad_long) - sizeof(long); #ifdef HAVE_LONG_LONG case 'q': case 'Q': return sizeof(__Pyx_pad_longlong) - sizeof(PY_LONG_LONG); #endif case 'f': return sizeof(__Pyx_pad_float) - sizeof(float); case 'd': return sizeof(__Pyx_pad_double) - sizeof(double); case 'g': return sizeof(__Pyx_pad_longdouble) - sizeof(long double); case 'P': case 'O': return sizeof(__Pyx_pad_void_p) - sizeof(void*); default: __Pyx_BufFmt_RaiseUnexpectedChar(ch); return 0; } } static char __Pyx_BufFmt_TypeCharToGroup(char ch, int is_complex) { switch (ch) { case 'c': return 'H'; case 'b': case 'h': case 'i': case 'l': case 'q': case 's': case 'p': return 'I'; case 'B': case 'H': case 'I': case 'L': case 'Q': return 'U'; case 'f': case 'd': case 'g': return (is_complex ? 'C' : 'R'); case 'O': return 'O'; case 'P': return 'P'; default: { __Pyx_BufFmt_RaiseUnexpectedChar(ch); return 0; } } } static void __Pyx_BufFmt_RaiseExpected(__Pyx_BufFmt_Context* ctx) { if (ctx->head == NULL || ctx->head->field == &ctx->root) { const char* expected; const char* quote; if (ctx->head == NULL) { expected = "end"; quote = ""; } else { expected = ctx->head->field->type->name; quote = "'"; } PyErr_Format(PyExc_ValueError, "Buffer dtype mismatch, expected %s%s%s but got %s", quote, expected, quote, __Pyx_BufFmt_DescribeTypeChar(ctx->enc_type, ctx->is_complex)); } else { __Pyx_StructField* field = ctx->head->field; __Pyx_StructField* parent = (ctx->head - 1)->field; PyErr_Format(PyExc_ValueError, "Buffer dtype mismatch, expected '%s' but got %s in '%s.%s'", field->type->name, __Pyx_BufFmt_DescribeTypeChar(ctx->enc_type, ctx->is_complex), parent->type->name, field->name); } } static int __Pyx_BufFmt_ProcessTypeChunk(__Pyx_BufFmt_Context* ctx) { char group; size_t size, offset, arraysize = 1; if (ctx->enc_type == 0) return 0; if (ctx->head->field->type->arraysize[0]) { int i, ndim = 0; if (ctx->enc_type == 's' || ctx->enc_type == 'p') { ctx->is_valid_array = ctx->head->field->type->ndim == 1; ndim = 1; if (ctx->enc_count != ctx->head->field->type->arraysize[0]) { PyErr_Format(PyExc_ValueError, "Expected a dimension of size %zu, got %zu", ctx->head->field->type->arraysize[0], ctx->enc_count); return -1; } } if (!ctx->is_valid_array) { PyErr_Format(PyExc_ValueError, "Expected %d dimensions, got %d", ctx->head->field->type->ndim, ndim); return -1; } for (i = 0; i < ctx->head->field->type->ndim; i++) { arraysize *= ctx->head->field->type->arraysize[i]; } ctx->is_valid_array = 0; ctx->enc_count = 1; } group = __Pyx_BufFmt_TypeCharToGroup(ctx->enc_type, ctx->is_complex); do { __Pyx_StructField* field = ctx->head->field; __Pyx_TypeInfo* type = field->type; if (ctx->enc_packmode == '@' || ctx->enc_packmode == '^') { size = __Pyx_BufFmt_TypeCharToNativeSize(ctx->enc_type, ctx->is_complex); } else { size = __Pyx_BufFmt_TypeCharToStandardSize(ctx->enc_type, ctx->is_complex); } if (ctx->enc_packmode == '@') { size_t align_at = __Pyx_BufFmt_TypeCharToAlignment(ctx->enc_type, ctx->is_complex); size_t align_mod_offset; if (align_at == 0) return -1; align_mod_offset = ctx->fmt_offset % align_at; if (align_mod_offset > 0) ctx->fmt_offset += align_at - align_mod_offset; if (ctx->struct_alignment == 0) ctx->struct_alignment = __Pyx_BufFmt_TypeCharToPadding(ctx->enc_type, ctx->is_complex); } if (type->size != size || type->typegroup != group) { if (type->typegroup == 'C' && type->fields != NULL) { size_t parent_offset = ctx->head->parent_offset + field->offset; ++ctx->head; ctx->head->field = type->fields; ctx->head->parent_offset = parent_offset; continue; } if ((type->typegroup == 'H' || group == 'H') && type->size == size) { } else { __Pyx_BufFmt_RaiseExpected(ctx); return -1; } } offset = ctx->head->parent_offset + field->offset; if (ctx->fmt_offset != offset) { PyErr_Format(PyExc_ValueError, "Buffer dtype mismatch; next field is at offset %" CYTHON_FORMAT_SSIZE_T "d but %" CYTHON_FORMAT_SSIZE_T "d expected", (Py_ssize_t)ctx->fmt_offset, (Py_ssize_t)offset); return -1; } ctx->fmt_offset += size; if (arraysize) ctx->fmt_offset += (arraysize - 1) * size; --ctx->enc_count; while (1) { if (field == &ctx->root) { ctx->head = NULL; if (ctx->enc_count != 0) { __Pyx_BufFmt_RaiseExpected(ctx); return -1; } break; } ctx->head->field = ++field; if (field->type == NULL) { --ctx->head; field = ctx->head->field; continue; } else if (field->type->typegroup == 'S') { size_t parent_offset = ctx->head->parent_offset + field->offset; if (field->type->fields->type == NULL) continue; field = field->type->fields; ++ctx->head; ctx->head->field = field; ctx->head->parent_offset = parent_offset; break; } else { break; } } } while (ctx->enc_count); ctx->enc_type = 0; ctx->is_complex = 0; return 0; } static CYTHON_INLINE PyObject * __pyx_buffmt_parse_array(__Pyx_BufFmt_Context* ctx, const char** tsp) { const char *ts = *tsp; int i = 0, number; int ndim = ctx->head->field->type->ndim; ; ++ts; if (ctx->new_count != 1) { PyErr_SetString(PyExc_ValueError, "Cannot handle repeated arrays in format string"); return NULL; } if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; while (*ts && *ts != ')') { switch (*ts) { case ' ': case '\f': case '\r': case '\n': case '\t': case '\v': continue; default: break; } number = __Pyx_BufFmt_ExpectNumber(&ts); if (number == -1) return NULL; if (i < ndim && (size_t) number != ctx->head->field->type->arraysize[i]) return PyErr_Format(PyExc_ValueError, "Expected a dimension of size %zu, got %d", ctx->head->field->type->arraysize[i], number); if (*ts != ',' && *ts != ')') return PyErr_Format(PyExc_ValueError, "Expected a comma in format string, got '%c'", *ts); if (*ts == ',') ts++; i++; } if (i != ndim) return PyErr_Format(PyExc_ValueError, "Expected %d dimension(s), got %d", ctx->head->field->type->ndim, i); if (!*ts) { PyErr_SetString(PyExc_ValueError, "Unexpected end of format string, expected ')'"); return NULL; } ctx->is_valid_array = 1; ctx->new_count = 1; *tsp = ++ts; return Py_None; } static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const char* ts) { int got_Z = 0; while (1) { switch(*ts) { case 0: if (ctx->enc_type != 0 && ctx->head == NULL) { __Pyx_BufFmt_RaiseExpected(ctx); return NULL; } if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; if (ctx->head != NULL) { __Pyx_BufFmt_RaiseExpected(ctx); return NULL; } return ts; case ' ': case '\r': case '\n': ++ts; break; case '<': if (!__Pyx_IsLittleEndian()) { PyErr_SetString(PyExc_ValueError, "Little-endian buffer not supported on big-endian compiler"); return NULL; } ctx->new_packmode = '='; ++ts; break; case '>': case '!': if (__Pyx_IsLittleEndian()) { PyErr_SetString(PyExc_ValueError, "Big-endian buffer not supported on little-endian compiler"); return NULL; } ctx->new_packmode = '='; ++ts; break; case '=': case '@': case '^': ctx->new_packmode = *ts++; break; case 'T': { const char* ts_after_sub; size_t i, struct_count = ctx->new_count; size_t struct_alignment = ctx->struct_alignment; ctx->new_count = 1; ++ts; if (*ts != '{') { PyErr_SetString(PyExc_ValueError, "Buffer acquisition: Expected '{' after 'T'"); return NULL; } if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; ctx->enc_type = 0; ctx->enc_count = 0; ctx->struct_alignment = 0; ++ts; ts_after_sub = ts; for (i = 0; i != struct_count; ++i) { ts_after_sub = __Pyx_BufFmt_CheckString(ctx, ts); if (!ts_after_sub) return NULL; } ts = ts_after_sub; if (struct_alignment) ctx->struct_alignment = struct_alignment; } break; case '}': { size_t alignment = ctx->struct_alignment; ++ts; if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; ctx->enc_type = 0; if (alignment && ctx->fmt_offset % alignment) { ctx->fmt_offset += alignment - (ctx->fmt_offset % alignment); } } return ts; case 'x': if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; ctx->fmt_offset += ctx->new_count; ctx->new_count = 1; ctx->enc_count = 0; ctx->enc_type = 0; ctx->enc_packmode = ctx->new_packmode; ++ts; break; case 'Z': got_Z = 1; ++ts; if (*ts != 'f' && *ts != 'd' && *ts != 'g') { __Pyx_BufFmt_RaiseUnexpectedChar('Z'); return NULL; } case 'c': case 'b': case 'B': case 'h': case 'H': case 'i': case 'I': case 'l': case 'L': case 'q': case 'Q': case 'f': case 'd': case 'g': case 'O': case 'p': if (ctx->enc_type == *ts && got_Z == ctx->is_complex && ctx->enc_packmode == ctx->new_packmode) { ctx->enc_count += ctx->new_count; ctx->new_count = 1; got_Z = 0; ++ts; break; } case 's': if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; ctx->enc_count = ctx->new_count; ctx->enc_packmode = ctx->new_packmode; ctx->enc_type = *ts; ctx->is_complex = got_Z; ++ts; ctx->new_count = 1; got_Z = 0; break; case ':': ++ts; while(*ts != ':') ++ts; ++ts; break; case '(': if (!__pyx_buffmt_parse_array(ctx, &ts)) return NULL; break; default: { int number = __Pyx_BufFmt_ExpectNumber(&ts); if (number == -1) return NULL; ctx->new_count = (size_t)number; } } } } static CYTHON_INLINE void __Pyx_ZeroBuffer(Py_buffer* buf) { buf->buf = NULL; buf->obj = NULL; buf->strides = __Pyx_zeros; buf->shape = __Pyx_zeros; buf->suboffsets = __Pyx_minusones; } static CYTHON_INLINE int __Pyx_GetBufferAndValidate( Py_buffer* buf, PyObject* obj, __Pyx_TypeInfo* dtype, int flags, int nd, int cast, __Pyx_BufFmt_StackElem* stack) { if (obj == Py_None || obj == NULL) { __Pyx_ZeroBuffer(buf); return 0; } buf->buf = NULL; if (__Pyx_GetBuffer(obj, buf, flags) == -1) goto fail; if (buf->ndim != nd) { PyErr_Format(PyExc_ValueError, "Buffer has wrong number of dimensions (expected %d, got %d)", nd, buf->ndim); goto fail; } if (!cast) { __Pyx_BufFmt_Context ctx; __Pyx_BufFmt_Init(&ctx, stack, dtype); if (!__Pyx_BufFmt_CheckString(&ctx, buf->format)) goto fail; } if ((unsigned)buf->itemsize != dtype->size) { PyErr_Format(PyExc_ValueError, "Item size of buffer (%" CYTHON_FORMAT_SSIZE_T "d byte%s) does not match size of '%s' (%" CYTHON_FORMAT_SSIZE_T "d byte%s)", buf->itemsize, (buf->itemsize > 1) ? "s" : "", dtype->name, (Py_ssize_t)dtype->size, (dtype->size > 1) ? "s" : ""); goto fail; } if (buf->suboffsets == NULL) buf->suboffsets = __Pyx_minusones; return 0; fail:; __Pyx_ZeroBuffer(buf); return -1; } static CYTHON_INLINE void __Pyx_SafeReleaseBuffer(Py_buffer* info) { if (info->buf == NULL) return; if (info->suboffsets == __Pyx_minusones) info->suboffsets = NULL; __Pyx_ReleaseBuffer(info); } /* GetBuiltinName */ static PyObject *__Pyx_GetBuiltinName(PyObject *name) { PyObject* result = __Pyx_PyObject_GetAttrStr(__pyx_b, name); if (unlikely(!result)) { PyErr_Format(PyExc_NameError, #if PY_MAJOR_VERSION >= 3 "name '%U' is not defined", name); #else "name '%.200s' is not defined", PyString_AS_STRING(name)); #endif } return result; } /* GetModuleGlobalName */ static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { PyObject *result; #if CYTHON_COMPILING_IN_CPYTHON result = PyDict_GetItem(__pyx_d, name); if (likely(result)) { Py_INCREF(result); } else { #else result = PyObject_GetItem(__pyx_d, name); if (!result) { PyErr_Clear(); #endif result = __Pyx_GetBuiltinName(name); } return result; } /* PyObjectCall */ #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { PyObject *result; ternaryfunc call = func->ob_type->tp_call; if (unlikely(!call)) return PyObject_Call(func, arg, kw); if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) return NULL; result = (*call)(func, arg, kw); Py_LeaveRecursiveCall(); if (unlikely(!result) && unlikely(!PyErr_Occurred())) { PyErr_SetString( PyExc_SystemError, "NULL result without error in PyObject_Call"); } return result; } #endif /* ExtTypeTest */ static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type) { if (unlikely(!type)) { PyErr_SetString(PyExc_SystemError, "Missing type object"); return 0; } if (likely(PyObject_TypeCheck(obj, type))) return 1; PyErr_Format(PyExc_TypeError, "Cannot convert %.200s to %.200s", Py_TYPE(obj)->tp_name, type->tp_name); return 0; } /* PyObjectCallMethO */ #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { PyObject *self, *result; PyCFunction cfunc; cfunc = PyCFunction_GET_FUNCTION(func); self = PyCFunction_GET_SELF(func); if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) return NULL; result = cfunc(self, arg); Py_LeaveRecursiveCall(); if (unlikely(!result) && unlikely(!PyErr_Occurred())) { PyErr_SetString( PyExc_SystemError, "NULL result without error in PyObject_Call"); } return result; } #endif /* PyObjectCallOneArg */ #if CYTHON_COMPILING_IN_CPYTHON static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { PyObject *result; PyObject *args = PyTuple_New(1); if (unlikely(!args)) return NULL; Py_INCREF(arg); PyTuple_SET_ITEM(args, 0, arg); result = __Pyx_PyObject_Call(func, args, NULL); Py_DECREF(args); return result; } static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { #ifdef __Pyx_CyFunction_USED if (likely(PyCFunction_Check(func) || PyObject_TypeCheck(func, __pyx_CyFunctionType))) { #else if (likely(PyCFunction_Check(func))) { #endif if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { return __Pyx_PyObject_CallMethO(func, arg); } } return __Pyx__PyObject_CallOneArg(func, arg); } #else static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { PyObject *result; PyObject *args = PyTuple_Pack(1, arg); if (unlikely(!args)) return NULL; result = __Pyx_PyObject_Call(func, args, NULL); Py_DECREF(args); return result; } #endif /* PyObjectCallNoArg */ #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { #ifdef __Pyx_CyFunction_USED if (likely(PyCFunction_Check(func) || PyObject_TypeCheck(func, __pyx_CyFunctionType))) { #else if (likely(PyCFunction_Check(func))) { #endif if (likely(PyCFunction_GET_FLAGS(func) & METH_NOARGS)) { return __Pyx_PyObject_CallMethO(func, NULL); } } return __Pyx_PyObject_Call(func, __pyx_empty_tuple, NULL); } #endif /* BufferIndexError */ static void __Pyx_RaiseBufferIndexError(int axis) { PyErr_Format(PyExc_IndexError, "Out of bounds on buffer access (axis %d)", axis); } /* SliceObject */ static CYTHON_INLINE PyObject* __Pyx_PyObject_GetSlice(PyObject* obj, Py_ssize_t cstart, Py_ssize_t cstop, PyObject** _py_start, PyObject** _py_stop, PyObject** _py_slice, int has_cstart, int has_cstop, CYTHON_UNUSED int wraparound) { #if CYTHON_COMPILING_IN_CPYTHON PyMappingMethods* mp; #if PY_MAJOR_VERSION < 3 PySequenceMethods* ms = Py_TYPE(obj)->tp_as_sequence; if (likely(ms && ms->sq_slice)) { if (!has_cstart) { if (_py_start && (*_py_start != Py_None)) { cstart = __Pyx_PyIndex_AsSsize_t(*_py_start); if ((cstart == (Py_ssize_t)-1) && PyErr_Occurred()) goto bad; } else cstart = 0; } if (!has_cstop) { if (_py_stop && (*_py_stop != Py_None)) { cstop = __Pyx_PyIndex_AsSsize_t(*_py_stop); if ((cstop == (Py_ssize_t)-1) && PyErr_Occurred()) goto bad; } else cstop = PY_SSIZE_T_MAX; } if (wraparound && unlikely((cstart < 0) | (cstop < 0)) && likely(ms->sq_length)) { Py_ssize_t l = ms->sq_length(obj); if (likely(l >= 0)) { if (cstop < 0) { cstop += l; if (cstop < 0) cstop = 0; } if (cstart < 0) { cstart += l; if (cstart < 0) cstart = 0; } } else { if (!PyErr_ExceptionMatches(PyExc_OverflowError)) goto bad; PyErr_Clear(); } } return ms->sq_slice(obj, cstart, cstop); } #endif mp = Py_TYPE(obj)->tp_as_mapping; if (likely(mp && mp->mp_subscript)) #endif { PyObject* result; PyObject *py_slice, *py_start, *py_stop; if (_py_slice) { py_slice = *_py_slice; } else { PyObject* owned_start = NULL; PyObject* owned_stop = NULL; if (_py_start) { py_start = *_py_start; } else { if (has_cstart) { owned_start = py_start = PyInt_FromSsize_t(cstart); if (unlikely(!py_start)) goto bad; } else py_start = Py_None; } if (_py_stop) { py_stop = *_py_stop; } else { if (has_cstop) { owned_stop = py_stop = PyInt_FromSsize_t(cstop); if (unlikely(!py_stop)) { Py_XDECREF(owned_start); goto bad; } } else py_stop = Py_None; } py_slice = PySlice_New(py_start, py_stop, Py_None); Py_XDECREF(owned_start); Py_XDECREF(owned_stop); if (unlikely(!py_slice)) goto bad; } #if CYTHON_COMPILING_IN_CPYTHON result = mp->mp_subscript(obj, py_slice); #else result = PyObject_GetItem(obj, py_slice); #endif if (!_py_slice) { Py_DECREF(py_slice); } return result; } PyErr_Format(PyExc_TypeError, "'%.200s' object is unsliceable", Py_TYPE(obj)->tp_name); bad: return NULL; } /* BufferFallbackError */ static void __Pyx_RaiseBufferFallbackError(void) { PyErr_SetString(PyExc_ValueError, "Buffer acquisition failed on assignment; and then reacquiring the old buffer failed too!"); } /* PyErrFetchRestore */ #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { PyObject *tmp_type, *tmp_value, *tmp_tb; tmp_type = tstate->curexc_type; tmp_value = tstate->curexc_value; tmp_tb = tstate->curexc_traceback; tstate->curexc_type = type; tstate->curexc_value = value; tstate->curexc_traceback = tb; Py_XDECREF(tmp_type); Py_XDECREF(tmp_value); Py_XDECREF(tmp_tb); } static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { *type = tstate->curexc_type; *value = tstate->curexc_value; *tb = tstate->curexc_traceback; tstate->curexc_type = 0; tstate->curexc_value = 0; tstate->curexc_traceback = 0; } #endif /* RaiseException */ #if PY_MAJOR_VERSION < 3 static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, CYTHON_UNUSED PyObject *cause) { __Pyx_PyThreadState_declare Py_XINCREF(type); if (!value || value == Py_None) value = NULL; else Py_INCREF(value); if (!tb || tb == Py_None) tb = NULL; else { Py_INCREF(tb); if (!PyTraceBack_Check(tb)) { PyErr_SetString(PyExc_TypeError, "raise: arg 3 must be a traceback or None"); goto raise_error; } } if (PyType_Check(type)) { #if CYTHON_COMPILING_IN_PYPY if (!value) { Py_INCREF(Py_None); value = Py_None; } #endif PyErr_NormalizeException(&type, &value, &tb); } else { if (value) { PyErr_SetString(PyExc_TypeError, "instance exception may not have a separate value"); goto raise_error; } value = type; type = (PyObject*) Py_TYPE(type); Py_INCREF(type); if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { PyErr_SetString(PyExc_TypeError, "raise: exception class must be a subclass of BaseException"); goto raise_error; } } __Pyx_PyThreadState_assign __Pyx_ErrRestore(type, value, tb); return; raise_error: Py_XDECREF(value); Py_XDECREF(type); Py_XDECREF(tb); return; } #else static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { PyObject* owned_instance = NULL; if (tb == Py_None) { tb = 0; } else if (tb && !PyTraceBack_Check(tb)) { PyErr_SetString(PyExc_TypeError, "raise: arg 3 must be a traceback or None"); goto bad; } if (value == Py_None) value = 0; if (PyExceptionInstance_Check(type)) { if (value) { PyErr_SetString(PyExc_TypeError, "instance exception may not have a separate value"); goto bad; } value = type; type = (PyObject*) Py_TYPE(value); } else if (PyExceptionClass_Check(type)) { PyObject *instance_class = NULL; if (value && PyExceptionInstance_Check(value)) { instance_class = (PyObject*) Py_TYPE(value); if (instance_class != type) { int is_subclass = PyObject_IsSubclass(instance_class, type); if (!is_subclass) { instance_class = NULL; } else if (unlikely(is_subclass == -1)) { goto bad; } else { type = instance_class; } } } if (!instance_class) { PyObject *args; if (!value) args = PyTuple_New(0); else if (PyTuple_Check(value)) { Py_INCREF(value); args = value; } else args = PyTuple_Pack(1, value); if (!args) goto bad; owned_instance = PyObject_Call(type, args, NULL); Py_DECREF(args); if (!owned_instance) goto bad; value = owned_instance; if (!PyExceptionInstance_Check(value)) { PyErr_Format(PyExc_TypeError, "calling %R should have returned an instance of " "BaseException, not %R", type, Py_TYPE(value)); goto bad; } } } else { PyErr_SetString(PyExc_TypeError, "raise: exception class must be a subclass of BaseException"); goto bad; } #if PY_VERSION_HEX >= 0x03030000 if (cause) { #else if (cause && cause != Py_None) { #endif PyObject *fixed_cause; if (cause == Py_None) { fixed_cause = NULL; } else if (PyExceptionClass_Check(cause)) { fixed_cause = PyObject_CallObject(cause, NULL); if (fixed_cause == NULL) goto bad; } else if (PyExceptionInstance_Check(cause)) { fixed_cause = cause; Py_INCREF(fixed_cause); } else { PyErr_SetString(PyExc_TypeError, "exception causes must derive from " "BaseException"); goto bad; } PyException_SetCause(value, fixed_cause); } PyErr_SetObject(type, value); if (tb) { #if CYTHON_COMPILING_IN_PYPY PyObject *tmp_type, *tmp_value, *tmp_tb; PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); Py_INCREF(tb); PyErr_Restore(tmp_type, tmp_value, tb); Py_XDECREF(tmp_tb); #else PyThreadState *tstate = PyThreadState_GET(); PyObject* tmp_tb = tstate->curexc_traceback; if (tb != tmp_tb) { Py_INCREF(tb); tstate->curexc_traceback = tb; Py_XDECREF(tmp_tb); } #endif } bad: Py_XDECREF(owned_instance); return; } #endif /* RaiseTooManyValuesToUnpack */ static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected) { PyErr_Format(PyExc_ValueError, "too many values to unpack (expected %" CYTHON_FORMAT_SSIZE_T "d)", expected); } /* RaiseNeedMoreValuesToUnpack */ static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) { PyErr_Format(PyExc_ValueError, "need more than %" CYTHON_FORMAT_SSIZE_T "d value%.1s to unpack", index, (index == 1) ? "" : "s"); } /* RaiseNoneIterError */ static CYTHON_INLINE void __Pyx_RaiseNoneNotIterableError(void) { PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); } /* Import */ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { PyObject *empty_list = 0; PyObject *module = 0; PyObject *global_dict = 0; PyObject *empty_dict = 0; PyObject *list; #if PY_VERSION_HEX < 0x03030000 PyObject *py_import; py_import = __Pyx_PyObject_GetAttrStr(__pyx_b, __pyx_n_s_import); if (!py_import) goto bad; #endif if (from_list) list = from_list; else { empty_list = PyList_New(0); if (!empty_list) goto bad; list = empty_list; } global_dict = PyModule_GetDict(__pyx_m); if (!global_dict) goto bad; empty_dict = PyDict_New(); if (!empty_dict) goto bad; { #if PY_MAJOR_VERSION >= 3 if (level == -1) { if (strchr(__Pyx_MODULE_NAME, '.')) { #if PY_VERSION_HEX < 0x03030000 PyObject *py_level = PyInt_FromLong(1); if (!py_level) goto bad; module = PyObject_CallFunctionObjArgs(py_import, name, global_dict, empty_dict, list, py_level, NULL); Py_DECREF(py_level); #else module = PyImport_ImportModuleLevelObject( name, global_dict, empty_dict, list, 1); #endif if (!module) { if (!PyErr_ExceptionMatches(PyExc_ImportError)) goto bad; PyErr_Clear(); } } level = 0; } #endif if (!module) { #if PY_VERSION_HEX < 0x03030000 PyObject *py_level = PyInt_FromLong(level); if (!py_level) goto bad; module = PyObject_CallFunctionObjArgs(py_import, name, global_dict, empty_dict, list, py_level, NULL); Py_DECREF(py_level); #else module = PyImport_ImportModuleLevelObject( name, global_dict, empty_dict, list, level); #endif } } bad: #if PY_VERSION_HEX < 0x03030000 Py_XDECREF(py_import); #endif Py_XDECREF(empty_list); Py_XDECREF(empty_dict); return module; } /* CodeObjectCache */ static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { int start = 0, mid = 0, end = count - 1; if (end >= 0 && code_line > entries[end].code_line) { return count; } while (start < end) { mid = start + (end - start) / 2; if (code_line < entries[mid].code_line) { end = mid; } else if (code_line > entries[mid].code_line) { start = mid + 1; } else { return mid; } } if (code_line <= entries[mid].code_line) { return mid; } else { return mid + 1; } } static PyCodeObject *__pyx_find_code_object(int code_line) { PyCodeObject* code_object; int pos; if (unlikely(!code_line) || unlikely(!__pyx_code_cache.entries)) { return NULL; } pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); if (unlikely(pos >= __pyx_code_cache.count) || unlikely(__pyx_code_cache.entries[pos].code_line != code_line)) { return NULL; } code_object = __pyx_code_cache.entries[pos].code_object; Py_INCREF(code_object); return code_object; } static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { int pos, i; __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries; if (unlikely(!code_line)) { return; } if (unlikely(!entries)) { entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); if (likely(entries)) { __pyx_code_cache.entries = entries; __pyx_code_cache.max_count = 64; __pyx_code_cache.count = 1; entries[0].code_line = code_line; entries[0].code_object = code_object; Py_INCREF(code_object); } return; } pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); if ((pos < __pyx_code_cache.count) && unlikely(__pyx_code_cache.entries[pos].code_line == code_line)) { PyCodeObject* tmp = entries[pos].code_object; entries[pos].code_object = code_object; Py_DECREF(tmp); return; } if (__pyx_code_cache.count == __pyx_code_cache.max_count) { int new_max = __pyx_code_cache.max_count + 64; entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( __pyx_code_cache.entries, (size_t)new_max*sizeof(__Pyx_CodeObjectCacheEntry)); if (unlikely(!entries)) { return; } __pyx_code_cache.entries = entries; __pyx_code_cache.max_count = new_max; } for (i=__pyx_code_cache.count; i>pos; i--) { entries[i] = entries[i-1]; } entries[pos].code_line = code_line; entries[pos].code_object = code_object; __pyx_code_cache.count++; Py_INCREF(code_object); } /* AddTraceback */ #include "compile.h" #include "frameobject.h" #include "traceback.h" static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( const char *funcname, int c_line, int py_line, const char *filename) { PyCodeObject *py_code = 0; PyObject *py_srcfile = 0; PyObject *py_funcname = 0; #if PY_MAJOR_VERSION < 3 py_srcfile = PyString_FromString(filename); #else py_srcfile = PyUnicode_FromString(filename); #endif if (!py_srcfile) goto bad; if (c_line) { #if PY_MAJOR_VERSION < 3 py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); #else py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); #endif } else { #if PY_MAJOR_VERSION < 3 py_funcname = PyString_FromString(funcname); #else py_funcname = PyUnicode_FromString(funcname); #endif } if (!py_funcname) goto bad; py_code = __Pyx_PyCode_New( 0, 0, 0, 0, 0, __pyx_empty_bytes, /*PyObject *code,*/ __pyx_empty_tuple, /*PyObject *consts,*/ __pyx_empty_tuple, /*PyObject *names,*/ __pyx_empty_tuple, /*PyObject *varnames,*/ __pyx_empty_tuple, /*PyObject *freevars,*/ __pyx_empty_tuple, /*PyObject *cellvars,*/ py_srcfile, /*PyObject *filename,*/ py_funcname, /*PyObject *name,*/ py_line, __pyx_empty_bytes /*PyObject *lnotab*/ ); Py_DECREF(py_srcfile); Py_DECREF(py_funcname); return py_code; bad: Py_XDECREF(py_srcfile); Py_XDECREF(py_funcname); return NULL; } static void __Pyx_AddTraceback(const char *funcname, int c_line, int py_line, const char *filename) { PyCodeObject *py_code = 0; PyFrameObject *py_frame = 0; py_code = __pyx_find_code_object(c_line ? c_line : py_line); if (!py_code) { py_code = __Pyx_CreateCodeObjectForTraceback( funcname, c_line, py_line, filename); if (!py_code) goto bad; __pyx_insert_code_object(c_line ? c_line : py_line, py_code); } py_frame = PyFrame_New( PyThreadState_GET(), /*PyThreadState *tstate,*/ py_code, /*PyCodeObject *code,*/ __pyx_d, /*PyObject *globals,*/ 0 /*PyObject *locals*/ ); if (!py_frame) goto bad; py_frame->f_lineno = py_line; PyTraceBack_Here(py_frame); bad: Py_XDECREF(py_code); Py_XDECREF(py_frame); } #if PY_MAJOR_VERSION < 3 static int __Pyx_GetBuffer(PyObject *obj, Py_buffer *view, int flags) { if (PyObject_CheckBuffer(obj)) return PyObject_GetBuffer(obj, view, flags); if (PyObject_TypeCheck(obj, __pyx_ptype_5numpy_ndarray)) return __pyx_pw_5numpy_7ndarray_1__getbuffer__(obj, view, flags); PyErr_Format(PyExc_TypeError, "'%.200s' does not have the buffer interface", Py_TYPE(obj)->tp_name); return -1; } static void __Pyx_ReleaseBuffer(Py_buffer *view) { PyObject *obj = view->obj; if (!obj) return; if (PyObject_CheckBuffer(obj)) { PyBuffer_Release(view); return; } if (PyObject_TypeCheck(obj, __pyx_ptype_5numpy_ndarray)) { __pyx_pw_5numpy_7ndarray_3__releasebuffer__(obj, view); return; } Py_DECREF(obj); view->obj = NULL; } #endif /* CIntFromPyVerify */ #define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) #define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) #define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ {\ func_type value = func_value;\ if (sizeof(target_type) < sizeof(func_type)) {\ if (unlikely(value != (func_type) (target_type) value)) {\ func_type zero = 0;\ if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ return (target_type) -1;\ if (is_unsigned && unlikely(value < zero))\ goto raise_neg_overflow;\ else\ goto raise_overflow;\ }\ }\ return (target_type) value;\ } /* CIntToPy */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) { const int neg_one = (int) -1, const_zero = (int) 0; const int is_unsigned = neg_one > const_zero; if (is_unsigned) { if (sizeof(int) < sizeof(long)) { return PyInt_FromLong((long) value); } else if (sizeof(int) <= sizeof(unsigned long)) { return PyLong_FromUnsignedLong((unsigned long) value); } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); } } else { if (sizeof(int) <= sizeof(long)) { return PyInt_FromLong((long) value); } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { return PyLong_FromLongLong((PY_LONG_LONG) value); } } { int one = 1; int little = (int)*(unsigned char *)&one; unsigned char *bytes = (unsigned char *)&value; return _PyLong_FromByteArray(bytes, sizeof(int), little, !is_unsigned); } } /* None */ #if CYTHON_CCOMPLEX #ifdef __cplusplus static CYTHON_INLINE __pyx_t_float_complex __pyx_t_float_complex_from_parts(float x, float y) { return ::std::complex< float >(x, y); } #else static CYTHON_INLINE __pyx_t_float_complex __pyx_t_float_complex_from_parts(float x, float y) { return x + y*(__pyx_t_float_complex)_Complex_I; } #endif #else static CYTHON_INLINE __pyx_t_float_complex __pyx_t_float_complex_from_parts(float x, float y) { __pyx_t_float_complex z; z.real = x; z.imag = y; return z; } #endif /* None */ #if CYTHON_CCOMPLEX #else static CYTHON_INLINE int __Pyx_c_eqf(__pyx_t_float_complex a, __pyx_t_float_complex b) { return (a.real == b.real) && (a.imag == b.imag); } static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_sumf(__pyx_t_float_complex a, __pyx_t_float_complex b) { __pyx_t_float_complex z; z.real = a.real + b.real; z.imag = a.imag + b.imag; return z; } static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_difff(__pyx_t_float_complex a, __pyx_t_float_complex b) { __pyx_t_float_complex z; z.real = a.real - b.real; z.imag = a.imag - b.imag; return z; } static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_prodf(__pyx_t_float_complex a, __pyx_t_float_complex b) { __pyx_t_float_complex z; z.real = a.real * b.real - a.imag * b.imag; z.imag = a.real * b.imag + a.imag * b.real; return z; } static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_quotf(__pyx_t_float_complex a, __pyx_t_float_complex b) { __pyx_t_float_complex z; float denom = b.real * b.real + b.imag * b.imag; z.real = (a.real * b.real + a.imag * b.imag) / denom; z.imag = (a.imag * b.real - a.real * b.imag) / denom; return z; } static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_negf(__pyx_t_float_complex a) { __pyx_t_float_complex z; z.real = -a.real; z.imag = -a.imag; return z; } static CYTHON_INLINE int __Pyx_c_is_zerof(__pyx_t_float_complex a) { return (a.real == 0) && (a.imag == 0); } static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_conjf(__pyx_t_float_complex a) { __pyx_t_float_complex z; z.real = a.real; z.imag = -a.imag; return z; } #if 1 static CYTHON_INLINE float __Pyx_c_absf(__pyx_t_float_complex z) { #if !defined(HAVE_HYPOT) || defined(_MSC_VER) return sqrtf(z.real*z.real + z.imag*z.imag); #else return hypotf(z.real, z.imag); #endif } static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_powf(__pyx_t_float_complex a, __pyx_t_float_complex b) { __pyx_t_float_complex z; float r, lnr, theta, z_r, z_theta; if (b.imag == 0 && b.real == (int)b.real) { if (b.real < 0) { float denom = a.real * a.real + a.imag * a.imag; a.real = a.real / denom; a.imag = -a.imag / denom; b.real = -b.real; } switch ((int)b.real) { case 0: z.real = 1; z.imag = 0; return z; case 1: return a; case 2: z = __Pyx_c_prodf(a, a); return __Pyx_c_prodf(a, a); case 3: z = __Pyx_c_prodf(a, a); return __Pyx_c_prodf(z, a); case 4: z = __Pyx_c_prodf(a, a); return __Pyx_c_prodf(z, z); } } if (a.imag == 0) { if (a.real == 0) { return a; } r = a.real; theta = 0; } else { r = __Pyx_c_absf(a); theta = atan2f(a.imag, a.real); } lnr = logf(r); z_r = expf(lnr * b.real - theta * b.imag); z_theta = theta * b.real + lnr * b.imag; z.real = z_r * cosf(z_theta); z.imag = z_r * sinf(z_theta); return z; } #endif #endif /* None */ #if CYTHON_CCOMPLEX #ifdef __cplusplus static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(double x, double y) { return ::std::complex< double >(x, y); } #else static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(double x, double y) { return x + y*(__pyx_t_double_complex)_Complex_I; } #endif #else static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(double x, double y) { __pyx_t_double_complex z; z.real = x; z.imag = y; return z; } #endif /* None */ #if CYTHON_CCOMPLEX #else static CYTHON_INLINE int __Pyx_c_eq(__pyx_t_double_complex a, __pyx_t_double_complex b) { return (a.real == b.real) && (a.imag == b.imag); } static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_sum(__pyx_t_double_complex a, __pyx_t_double_complex b) { __pyx_t_double_complex z; z.real = a.real + b.real; z.imag = a.imag + b.imag; return z; } static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_diff(__pyx_t_double_complex a, __pyx_t_double_complex b) { __pyx_t_double_complex z; z.real = a.real - b.real; z.imag = a.imag - b.imag; return z; } static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_prod(__pyx_t_double_complex a, __pyx_t_double_complex b) { __pyx_t_double_complex z; z.real = a.real * b.real - a.imag * b.imag; z.imag = a.real * b.imag + a.imag * b.real; return z; } static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_quot(__pyx_t_double_complex a, __pyx_t_double_complex b) { __pyx_t_double_complex z; double denom = b.real * b.real + b.imag * b.imag; z.real = (a.real * b.real + a.imag * b.imag) / denom; z.imag = (a.imag * b.real - a.real * b.imag) / denom; return z; } static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_neg(__pyx_t_double_complex a) { __pyx_t_double_complex z; z.real = -a.real; z.imag = -a.imag; return z; } static CYTHON_INLINE int __Pyx_c_is_zero(__pyx_t_double_complex a) { return (a.real == 0) && (a.imag == 0); } static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_conj(__pyx_t_double_complex a) { __pyx_t_double_complex z; z.real = a.real; z.imag = -a.imag; return z; } #if 1 static CYTHON_INLINE double __Pyx_c_abs(__pyx_t_double_complex z) { #if !defined(HAVE_HYPOT) || defined(_MSC_VER) return sqrt(z.real*z.real + z.imag*z.imag); #else return hypot(z.real, z.imag); #endif } static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_pow(__pyx_t_double_complex a, __pyx_t_double_complex b) { __pyx_t_double_complex z; double r, lnr, theta, z_r, z_theta; if (b.imag == 0 && b.real == (int)b.real) { if (b.real < 0) { double denom = a.real * a.real + a.imag * a.imag; a.real = a.real / denom; a.imag = -a.imag / denom; b.real = -b.real; } switch ((int)b.real) { case 0: z.real = 1; z.imag = 0; return z; case 1: return a; case 2: z = __Pyx_c_prod(a, a); return __Pyx_c_prod(a, a); case 3: z = __Pyx_c_prod(a, a); return __Pyx_c_prod(z, a); case 4: z = __Pyx_c_prod(a, a); return __Pyx_c_prod(z, z); } } if (a.imag == 0) { if (a.real == 0) { return a; } r = a.real; theta = 0; } else { r = __Pyx_c_abs(a); theta = atan2(a.imag, a.real); } lnr = log(r); z_r = exp(lnr * b.real - theta * b.imag); z_theta = theta * b.real + lnr * b.imag; z.real = z_r * cos(z_theta); z.imag = z_r * sin(z_theta); return z; } #endif #endif /* CIntToPy */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_enum__NPY_TYPES(enum NPY_TYPES value) { const enum NPY_TYPES neg_one = (enum NPY_TYPES) -1, const_zero = (enum NPY_TYPES) 0; const int is_unsigned = neg_one > const_zero; if (is_unsigned) { if (sizeof(enum NPY_TYPES) < sizeof(long)) { return PyInt_FromLong((long) value); } else if (sizeof(enum NPY_TYPES) <= sizeof(unsigned long)) { return PyLong_FromUnsignedLong((unsigned long) value); } else if (sizeof(enum NPY_TYPES) <= sizeof(unsigned PY_LONG_LONG)) { return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); } } else { if (sizeof(enum NPY_TYPES) <= sizeof(long)) { return PyInt_FromLong((long) value); } else if (sizeof(enum NPY_TYPES) <= sizeof(PY_LONG_LONG)) { return PyLong_FromLongLong((PY_LONG_LONG) value); } } { int one = 1; int little = (int)*(unsigned char *)&one; unsigned char *bytes = (unsigned char *)&value; return _PyLong_FromByteArray(bytes, sizeof(enum NPY_TYPES), little, !is_unsigned); } } /* CIntFromPy */ static CYTHON_INLINE npy_int32 __Pyx_PyInt_As_npy_int32(PyObject *x) { const npy_int32 neg_one = (npy_int32) -1, const_zero = (npy_int32) 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { if (sizeof(npy_int32) < sizeof(long)) { __PYX_VERIFY_RETURN_INT(npy_int32, long, PyInt_AS_LONG(x)) } else { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { goto raise_neg_overflow; } return (npy_int32) val; } } else #endif if (likely(PyLong_Check(x))) { if (is_unsigned) { #if CYTHON_USE_PYLONG_INTERNALS const digit* digits = ((PyLongObject*)x)->ob_digit; switch (Py_SIZE(x)) { case 0: return (npy_int32) 0; case 1: __PYX_VERIFY_RETURN_INT(npy_int32, digit, digits[0]) case 2: if (8 * sizeof(npy_int32) > 1 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(npy_int32, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(npy_int32) >= 2 * PyLong_SHIFT) { return (npy_int32) (((((npy_int32)digits[1]) << PyLong_SHIFT) | (npy_int32)digits[0])); } } break; case 3: if (8 * sizeof(npy_int32) > 2 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(npy_int32, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(npy_int32) >= 3 * PyLong_SHIFT) { return (npy_int32) (((((((npy_int32)digits[2]) << PyLong_SHIFT) | (npy_int32)digits[1]) << PyLong_SHIFT) | (npy_int32)digits[0])); } } break; case 4: if (8 * sizeof(npy_int32) > 3 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(npy_int32, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(npy_int32) >= 4 * PyLong_SHIFT) { return (npy_int32) (((((((((npy_int32)digits[3]) << PyLong_SHIFT) | (npy_int32)digits[2]) << PyLong_SHIFT) | (npy_int32)digits[1]) << PyLong_SHIFT) | (npy_int32)digits[0])); } } break; } #endif #if CYTHON_COMPILING_IN_CPYTHON if (unlikely(Py_SIZE(x) < 0)) { goto raise_neg_overflow; } #else { int result = PyObject_RichCompareBool(x, Py_False, Py_LT); if (unlikely(result < 0)) return (npy_int32) -1; if (unlikely(result == 1)) goto raise_neg_overflow; } #endif if (sizeof(npy_int32) <= sizeof(unsigned long)) { __PYX_VERIFY_RETURN_INT_EXC(npy_int32, unsigned long, PyLong_AsUnsignedLong(x)) } else if (sizeof(npy_int32) <= sizeof(unsigned PY_LONG_LONG)) { __PYX_VERIFY_RETURN_INT_EXC(npy_int32, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) } } else { #if CYTHON_USE_PYLONG_INTERNALS const digit* digits = ((PyLongObject*)x)->ob_digit; switch (Py_SIZE(x)) { case 0: return (npy_int32) 0; case -1: __PYX_VERIFY_RETURN_INT(npy_int32, sdigit, (sdigit) (-(sdigit)digits[0])) case 1: __PYX_VERIFY_RETURN_INT(npy_int32, digit, +digits[0]) case -2: if (8 * sizeof(npy_int32) - 1 > 1 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(npy_int32, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(npy_int32) - 1 > 2 * PyLong_SHIFT) { return (npy_int32) (((npy_int32)-1)*(((((npy_int32)digits[1]) << PyLong_SHIFT) | (npy_int32)digits[0]))); } } break; case 2: if (8 * sizeof(npy_int32) > 1 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(npy_int32, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(npy_int32) - 1 > 2 * PyLong_SHIFT) { return (npy_int32) ((((((npy_int32)digits[1]) << PyLong_SHIFT) | (npy_int32)digits[0]))); } } break; case -3: if (8 * sizeof(npy_int32) - 1 > 2 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(npy_int32, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(npy_int32) - 1 > 3 * PyLong_SHIFT) { return (npy_int32) (((npy_int32)-1)*(((((((npy_int32)digits[2]) << PyLong_SHIFT) | (npy_int32)digits[1]) << PyLong_SHIFT) | (npy_int32)digits[0]))); } } break; case 3: if (8 * sizeof(npy_int32) > 2 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(npy_int32, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(npy_int32) - 1 > 3 * PyLong_SHIFT) { return (npy_int32) ((((((((npy_int32)digits[2]) << PyLong_SHIFT) | (npy_int32)digits[1]) << PyLong_SHIFT) | (npy_int32)digits[0]))); } } break; case -4: if (8 * sizeof(npy_int32) - 1 > 3 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(npy_int32, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(npy_int32) - 1 > 4 * PyLong_SHIFT) { return (npy_int32) (((npy_int32)-1)*(((((((((npy_int32)digits[3]) << PyLong_SHIFT) | (npy_int32)digits[2]) << PyLong_SHIFT) | (npy_int32)digits[1]) << PyLong_SHIFT) | (npy_int32)digits[0]))); } } break; case 4: if (8 * sizeof(npy_int32) > 3 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(npy_int32, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(npy_int32) - 1 > 4 * PyLong_SHIFT) { return (npy_int32) ((((((((((npy_int32)digits[3]) << PyLong_SHIFT) | (npy_int32)digits[2]) << PyLong_SHIFT) | (npy_int32)digits[1]) << PyLong_SHIFT) | (npy_int32)digits[0]))); } } break; } #endif if (sizeof(npy_int32) <= sizeof(long)) { __PYX_VERIFY_RETURN_INT_EXC(npy_int32, long, PyLong_AsLong(x)) } else if (sizeof(npy_int32) <= sizeof(PY_LONG_LONG)) { __PYX_VERIFY_RETURN_INT_EXC(npy_int32, PY_LONG_LONG, PyLong_AsLongLong(x)) } } { #if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) PyErr_SetString(PyExc_RuntimeError, "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); #else npy_int32 val; PyObject *v = __Pyx_PyNumber_IntOrLong(x); #if PY_MAJOR_VERSION < 3 if (likely(v) && !PyLong_Check(v)) { PyObject *tmp = v; v = PyNumber_Long(tmp); Py_DECREF(tmp); } #endif if (likely(v)) { int one = 1; int is_little = (int)*(unsigned char *)&one; unsigned char *bytes = (unsigned char *)&val; int ret = _PyLong_AsByteArray((PyLongObject *)v, bytes, sizeof(val), is_little, !is_unsigned); Py_DECREF(v); if (likely(!ret)) return val; } #endif return (npy_int32) -1; } } else { npy_int32 val; PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); if (!tmp) return (npy_int32) -1; val = __Pyx_PyInt_As_npy_int32(tmp); Py_DECREF(tmp); return val; } raise_overflow: PyErr_SetString(PyExc_OverflowError, "value too large to convert to npy_int32"); return (npy_int32) -1; raise_neg_overflow: PyErr_SetString(PyExc_OverflowError, "can't convert negative value to npy_int32"); return (npy_int32) -1; } /* CIntFromPy */ static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { const int neg_one = (int) -1, const_zero = (int) 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { if (sizeof(int) < sizeof(long)) { __PYX_VERIFY_RETURN_INT(int, long, PyInt_AS_LONG(x)) } else { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { goto raise_neg_overflow; } return (int) val; } } else #endif if (likely(PyLong_Check(x))) { if (is_unsigned) { #if CYTHON_USE_PYLONG_INTERNALS const digit* digits = ((PyLongObject*)x)->ob_digit; switch (Py_SIZE(x)) { case 0: return (int) 0; case 1: __PYX_VERIFY_RETURN_INT(int, digit, digits[0]) case 2: if (8 * sizeof(int) > 1 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(int) >= 2 * PyLong_SHIFT) { return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); } } break; case 3: if (8 * sizeof(int) > 2 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(int) >= 3 * PyLong_SHIFT) { return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); } } break; case 4: if (8 * sizeof(int) > 3 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(int) >= 4 * PyLong_SHIFT) { return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); } } break; } #endif #if CYTHON_COMPILING_IN_CPYTHON if (unlikely(Py_SIZE(x) < 0)) { goto raise_neg_overflow; } #else { int result = PyObject_RichCompareBool(x, Py_False, Py_LT); if (unlikely(result < 0)) return (int) -1; if (unlikely(result == 1)) goto raise_neg_overflow; } #endif if (sizeof(int) <= sizeof(unsigned long)) { __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) } } else { #if CYTHON_USE_PYLONG_INTERNALS const digit* digits = ((PyLongObject*)x)->ob_digit; switch (Py_SIZE(x)) { case 0: return (int) 0; case -1: __PYX_VERIFY_RETURN_INT(int, sdigit, (sdigit) (-(sdigit)digits[0])) case 1: __PYX_VERIFY_RETURN_INT(int, digit, +digits[0]) case -2: if (8 * sizeof(int) - 1 > 1 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); } } break; case 2: if (8 * sizeof(int) > 1 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); } } break; case -3: if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); } } break; case 3: if (8 * sizeof(int) > 2 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); } } break; case -4: if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); } } break; case 4: if (8 * sizeof(int) > 3 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); } } break; } #endif if (sizeof(int) <= sizeof(long)) { __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) } } { #if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) PyErr_SetString(PyExc_RuntimeError, "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); #else int val; PyObject *v = __Pyx_PyNumber_IntOrLong(x); #if PY_MAJOR_VERSION < 3 if (likely(v) && !PyLong_Check(v)) { PyObject *tmp = v; v = PyNumber_Long(tmp); Py_DECREF(tmp); } #endif if (likely(v)) { int one = 1; int is_little = (int)*(unsigned char *)&one; unsigned char *bytes = (unsigned char *)&val; int ret = _PyLong_AsByteArray((PyLongObject *)v, bytes, sizeof(val), is_little, !is_unsigned); Py_DECREF(v); if (likely(!ret)) return val; } #endif return (int) -1; } } else { int val; PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); if (!tmp) return (int) -1; val = __Pyx_PyInt_As_int(tmp); Py_DECREF(tmp); return val; } raise_overflow: PyErr_SetString(PyExc_OverflowError, "value too large to convert to int"); return (int) -1; raise_neg_overflow: PyErr_SetString(PyExc_OverflowError, "can't convert negative value to int"); return (int) -1; } /* CIntToPy */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { const long neg_one = (long) -1, const_zero = (long) 0; const int is_unsigned = neg_one > const_zero; if (is_unsigned) { if (sizeof(long) < sizeof(long)) { return PyInt_FromLong((long) value); } else if (sizeof(long) <= sizeof(unsigned long)) { return PyLong_FromUnsignedLong((unsigned long) value); } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); } } else { if (sizeof(long) <= sizeof(long)) { return PyInt_FromLong((long) value); } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { return PyLong_FromLongLong((PY_LONG_LONG) value); } } { int one = 1; int little = (int)*(unsigned char *)&one; unsigned char *bytes = (unsigned char *)&value; return _PyLong_FromByteArray(bytes, sizeof(long), little, !is_unsigned); } } /* CIntFromPy */ static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { const long neg_one = (long) -1, const_zero = (long) 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { if (sizeof(long) < sizeof(long)) { __PYX_VERIFY_RETURN_INT(long, long, PyInt_AS_LONG(x)) } else { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { goto raise_neg_overflow; } return (long) val; } } else #endif if (likely(PyLong_Check(x))) { if (is_unsigned) { #if CYTHON_USE_PYLONG_INTERNALS const digit* digits = ((PyLongObject*)x)->ob_digit; switch (Py_SIZE(x)) { case 0: return (long) 0; case 1: __PYX_VERIFY_RETURN_INT(long, digit, digits[0]) case 2: if (8 * sizeof(long) > 1 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(long) >= 2 * PyLong_SHIFT) { return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); } } break; case 3: if (8 * sizeof(long) > 2 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(long) >= 3 * PyLong_SHIFT) { return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); } } break; case 4: if (8 * sizeof(long) > 3 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(long) >= 4 * PyLong_SHIFT) { return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); } } break; } #endif #if CYTHON_COMPILING_IN_CPYTHON if (unlikely(Py_SIZE(x) < 0)) { goto raise_neg_overflow; } #else { int result = PyObject_RichCompareBool(x, Py_False, Py_LT); if (unlikely(result < 0)) return (long) -1; if (unlikely(result == 1)) goto raise_neg_overflow; } #endif if (sizeof(long) <= sizeof(unsigned long)) { __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) } } else { #if CYTHON_USE_PYLONG_INTERNALS const digit* digits = ((PyLongObject*)x)->ob_digit; switch (Py_SIZE(x)) { case 0: return (long) 0; case -1: __PYX_VERIFY_RETURN_INT(long, sdigit, (sdigit) (-(sdigit)digits[0])) case 1: __PYX_VERIFY_RETURN_INT(long, digit, +digits[0]) case -2: if (8 * sizeof(long) - 1 > 1 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); } } break; case 2: if (8 * sizeof(long) > 1 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); } } break; case -3: if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); } } break; case 3: if (8 * sizeof(long) > 2 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); } } break; case -4: if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); } } break; case 4: if (8 * sizeof(long) > 3 * PyLong_SHIFT) { if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); } } break; } #endif if (sizeof(long) <= sizeof(long)) { __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) } } { #if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) PyErr_SetString(PyExc_RuntimeError, "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); #else long val; PyObject *v = __Pyx_PyNumber_IntOrLong(x); #if PY_MAJOR_VERSION < 3 if (likely(v) && !PyLong_Check(v)) { PyObject *tmp = v; v = PyNumber_Long(tmp); Py_DECREF(tmp); } #endif if (likely(v)) { int one = 1; int is_little = (int)*(unsigned char *)&one; unsigned char *bytes = (unsigned char *)&val; int ret = _PyLong_AsByteArray((PyLongObject *)v, bytes, sizeof(val), is_little, !is_unsigned); Py_DECREF(v); if (likely(!ret)) return val; } #endif return (long) -1; } } else { long val; PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); if (!tmp) return (long) -1; val = __Pyx_PyInt_As_long(tmp); Py_DECREF(tmp); return val; } raise_overflow: PyErr_SetString(PyExc_OverflowError, "value too large to convert to long"); return (long) -1; raise_neg_overflow: PyErr_SetString(PyExc_OverflowError, "can't convert negative value to long"); return (long) -1; } /* CheckBinaryVersion */ static int __Pyx_check_binary_version(void) { char ctversion[4], rtversion[4]; PyOS_snprintf(ctversion, 4, "%d.%d", PY_MAJOR_VERSION, PY_MINOR_VERSION); PyOS_snprintf(rtversion, 4, "%s", Py_GetVersion()); if (ctversion[0] != rtversion[0] || ctversion[2] != rtversion[2]) { char message[200]; PyOS_snprintf(message, sizeof(message), "compiletime version %s of module '%.100s' " "does not match runtime version %s", ctversion, __Pyx_MODULE_NAME, rtversion); return PyErr_WarnEx(NULL, message, 1); } return 0; } /* ModuleImport */ #ifndef __PYX_HAVE_RT_ImportModule #define __PYX_HAVE_RT_ImportModule static PyObject *__Pyx_ImportModule(const char *name) { PyObject *py_name = 0; PyObject *py_module = 0; py_name = __Pyx_PyIdentifier_FromString(name); if (!py_name) goto bad; py_module = PyImport_Import(py_name); Py_DECREF(py_name); return py_module; bad: Py_XDECREF(py_name); return 0; } #endif /* TypeImport */ #ifndef __PYX_HAVE_RT_ImportType #define __PYX_HAVE_RT_ImportType static PyTypeObject *__Pyx_ImportType(const char *module_name, const char *class_name, size_t size, int strict) { PyObject *py_module = 0; PyObject *result = 0; PyObject *py_name = 0; char warning[200]; Py_ssize_t basicsize; #ifdef Py_LIMITED_API PyObject *py_basicsize; #endif py_module = __Pyx_ImportModule(module_name); if (!py_module) goto bad; py_name = __Pyx_PyIdentifier_FromString(class_name); if (!py_name) goto bad; result = PyObject_GetAttr(py_module, py_name); Py_DECREF(py_name); py_name = 0; Py_DECREF(py_module); py_module = 0; if (!result) goto bad; if (!PyType_Check(result)) { PyErr_Format(PyExc_TypeError, "%.200s.%.200s is not a type object", module_name, class_name); goto bad; } #ifndef Py_LIMITED_API basicsize = ((PyTypeObject *)result)->tp_basicsize; #else py_basicsize = PyObject_GetAttrString(result, "__basicsize__"); if (!py_basicsize) goto bad; basicsize = PyLong_AsSsize_t(py_basicsize); Py_DECREF(py_basicsize); py_basicsize = 0; if (basicsize == (Py_ssize_t)-1 && PyErr_Occurred()) goto bad; #endif if (!strict && (size_t)basicsize > size) { PyOS_snprintf(warning, sizeof(warning), "%s.%s size changed, may indicate binary incompatibility. Expected %zd, got %zd", module_name, class_name, basicsize, size); if (PyErr_WarnEx(NULL, warning, 0) < 0) goto bad; } else if ((size_t)basicsize != size) { PyErr_Format(PyExc_ValueError, "%.200s.%.200s has the wrong size, try recompiling. Expected %zd, got %zd", module_name, class_name, basicsize, size); goto bad; } return (PyTypeObject *)result; bad: Py_XDECREF(py_module); Py_XDECREF(result); return NULL; } #endif /* InitStrings */ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { while (t->p) { #if PY_MAJOR_VERSION < 3 if (t->is_unicode) { *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); } else if (t->intern) { *t->p = PyString_InternFromString(t->s); } else { *t->p = PyString_FromStringAndSize(t->s, t->n - 1); } #else if (t->is_unicode | t->is_str) { if (t->intern) { *t->p = PyUnicode_InternFromString(t->s); } else if (t->encoding) { *t->p = PyUnicode_Decode(t->s, t->n - 1, t->encoding, NULL); } else { *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); } } else { *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); } #endif if (!*t->p) return -1; ++t; } return 0; } static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { return __Pyx_PyUnicode_FromStringAndSize(c_str, (Py_ssize_t)strlen(c_str)); } static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject* o) { Py_ssize_t ignore; return __Pyx_PyObject_AsStringAndSize(o, &ignore); } static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { #if CYTHON_COMPILING_IN_CPYTHON && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) if ( #if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII __Pyx_sys_getdefaultencoding_not_ascii && #endif PyUnicode_Check(o)) { #if PY_VERSION_HEX < 0x03030000 char* defenc_c; PyObject* defenc = _PyUnicode_AsDefaultEncodedString(o, NULL); if (!defenc) return NULL; defenc_c = PyBytes_AS_STRING(defenc); #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII { char* end = defenc_c + PyBytes_GET_SIZE(defenc); char* c; for (c = defenc_c; c < end; c++) { if ((unsigned char) (*c) >= 128) { PyUnicode_AsASCIIString(o); return NULL; } } } #endif *length = PyBytes_GET_SIZE(defenc); return defenc_c; #else if (__Pyx_PyUnicode_READY(o) == -1) return NULL; #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII if (PyUnicode_IS_ASCII(o)) { *length = PyUnicode_GET_LENGTH(o); return PyUnicode_AsUTF8(o); } else { PyUnicode_AsASCIIString(o); return NULL; } #else return PyUnicode_AsUTF8AndSize(o, length); #endif #endif } else #endif #if (!CYTHON_COMPILING_IN_PYPY) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) if (PyByteArray_Check(o)) { *length = PyByteArray_GET_SIZE(o); return PyByteArray_AS_STRING(o); } else #endif { char* result; int r = PyBytes_AsStringAndSize(o, &result, length); if (unlikely(r < 0)) { return NULL; } else { return result; } } } static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { int is_true = x == Py_True; if (is_true | (x == Py_False) | (x == Py_None)) return is_true; else return PyObject_IsTrue(x); } static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x) { PyNumberMethods *m; const char *name = NULL; PyObject *res = NULL; #if PY_MAJOR_VERSION < 3 if (PyInt_Check(x) || PyLong_Check(x)) #else if (PyLong_Check(x)) #endif return __Pyx_NewRef(x); m = Py_TYPE(x)->tp_as_number; #if PY_MAJOR_VERSION < 3 if (m && m->nb_int) { name = "int"; res = PyNumber_Int(x); } else if (m && m->nb_long) { name = "long"; res = PyNumber_Long(x); } #else if (m && m->nb_int) { name = "int"; res = PyNumber_Long(x); } #endif if (res) { #if PY_MAJOR_VERSION < 3 if (!PyInt_Check(res) && !PyLong_Check(res)) { #else if (!PyLong_Check(res)) { #endif PyErr_Format(PyExc_TypeError, "__%.4s__ returned non-%.4s (type %.200s)", name, name, Py_TYPE(res)->tp_name); Py_DECREF(res); return NULL; } } else if (!PyErr_Occurred()) { PyErr_SetString(PyExc_TypeError, "an integer is required"); } return res; } static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { Py_ssize_t ival; PyObject *x; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_CheckExact(b))) { if (sizeof(Py_ssize_t) >= sizeof(long)) return PyInt_AS_LONG(b); else return PyInt_AsSsize_t(x); } #endif if (likely(PyLong_CheckExact(b))) { #if CYTHON_USE_PYLONG_INTERNALS const digit* digits = ((PyLongObject*)b)->ob_digit; const Py_ssize_t size = Py_SIZE(b); if (likely(__Pyx_sst_abs(size) <= 1)) { ival = likely(size) ? digits[0] : 0; if (size == -1) ival = -ival; return ival; } else { switch (size) { case 2: if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); } break; case -2: if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); } break; case 3: if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); } break; case -3: if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); } break; case 4: if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); } break; case -4: if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); } break; } } #endif return PyLong_AsSsize_t(b); } x = PyNumber_Index(b); if (!x) return -1; ival = PyInt_AsSsize_t(x); Py_DECREF(x); return ival; } static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { return PyInt_FromSize_t(ival); } #endif /* Py_PYTHON_H */ ================================================ FILE: lib/nms/gpu_nms.hpp ================================================ // ------------------------------------------------------------------ // Deep Feature Flow // Copyright (c) 2017 Microsoft // Licensed under The MIT License // Modified from MATLAB Faster R-CNN (https://github.com/shaoqingren/faster_rcnn) // ------------------------------------------------------------------ // Based on: // Faster R-CNN // Copyright (c) 2015 Microsoft // Licensed under The MIT License // https://github.com/shaoqingren/faster_rcnn // ------------------------------------------------------------------ void _nms(int* keep_out, int* num_out, const float* boxes_host, int boxes_num, int boxes_dim, float nms_overlap_thresh, int device_id); ================================================ FILE: lib/nms/gpu_nms.pyx ================================================ # ------------------------------------------------------------------ # Deep Feature Flow # Copyright (c) 2015 Microsoft # Licensed under The MIT License # Written by Yuwen Xiong, Tao Kong # ------------------------------------------------------------------ # Based on: # Faster R-CNN # Copyright (c) 2015 Microsoft # Licensed under The MIT License # https://github.com/shaoqingren/faster_rcnn # ------------------------------------------------------------------ import numpy as np cimport numpy as np assert sizeof(int) == sizeof(np.int32_t) cdef extern from "gpu_nms.hpp": void _nms(np.int32_t*, int*, np.float32_t*, int, int, float, int) def gpu_nms(np.ndarray[np.float32_t, ndim=2] dets, np.float thresh, np.int32_t device_id=0): cdef int boxes_num = dets.shape[0] cdef int boxes_dim = dets.shape[1] cdef int num_out cdef np.ndarray[np.int32_t, ndim=1] \ keep = np.zeros(boxes_num, dtype=np.int32) cdef np.ndarray[np.float32_t, ndim=1] \ scores = dets[:, 4] cdef np.ndarray[np.int32_t, ndim=1] \ order = scores.argsort()[::-1].astype(np.int32) cdef np.ndarray[np.float32_t, ndim=2] \ sorted_dets = dets[order, :] _nms(&keep[0], &num_out, &sorted_dets[0, 0], boxes_num, boxes_dim, thresh, device_id) keep = keep[:num_out] return list(order[keep]) ================================================ FILE: lib/nms/nms.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong # -------------------------------------------------------- # Based on: # py-faster-rcnn # Copyright (c) 2015 Microsoft # Licence under The MIT License # https://github.com/rbgirshick/py-faster-rcnn # -------------------------------------------------------- import numpy as np from cpu_nms import cpu_nms from gpu_nms import gpu_nms def py_nms_wrapper(thresh): def _nms(dets): return nms(dets, thresh) return _nms def cpu_nms_wrapper(thresh): def _nms(dets): return cpu_nms(dets, thresh) return _nms def gpu_nms_wrapper(thresh, device_id): def _nms(dets): return gpu_nms(dets, thresh, device_id) return _nms def nms(dets, thresh): """ greedily select boxes with high confidence and overlap with current maximum <= thresh rule out overlap >= thresh :param dets: [[x1, y1, x2, y2 score]] :param thresh: retain overlap < thresh :return: indexes to keep """ if dets.shape[0] == 0: return [] x1 = dets[:, 0] y1 = dets[:, 1] x2 = dets[:, 2] y2 = dets[:, 3] scores = dets[:, 4] areas = (x2 - x1 + 1) * (y2 - y1 + 1) order = scores.argsort()[::-1] keep = [] while order.size > 0: i = order[0] keep.append(i) xx1 = np.maximum(x1[i], x1[order[1:]]) yy1 = np.maximum(y1[i], y1[order[1:]]) xx2 = np.minimum(x2[i], x2[order[1:]]) yy2 = np.minimum(y2[i], y2[order[1:]]) w = np.maximum(0.0, xx2 - xx1 + 1) h = np.maximum(0.0, yy2 - yy1 + 1) inter = w * h ovr = inter / (areas[i] + areas[order[1:]] - inter) inds = np.where(ovr <= thresh)[0] order = order[inds + 1] return keep ================================================ FILE: lib/nms/nms_kernel.cu ================================================ // ------------------------------------------------------------------ // Deep Feature Flow // Copyright (c) 2015 Microsoft // Licensed under The MIT License // Written by Yuwen Xiong // ------------------------------------------------------------------ // Based on: // Faster R-CNN // Copyright (c) 2015 Microsoft // Licensed under The MIT License // https://github.com/shaoqingren/faster_rcnn // ------------------------------------------------------------------ #include "gpu_nms.hpp" #include #include #define CUDA_CHECK(condition) \ /* Code block avoids redefinition of cudaError_t error */ \ do { \ cudaError_t error = condition; \ if (error != cudaSuccess) { \ std::cout << cudaGetErrorString(error) << std::endl; \ } \ } while (0) #define DIVUP(m,n) ((m) / (n) + ((m) % (n) > 0)) int const threadsPerBlock = sizeof(unsigned long long) * 8; __device__ inline float devIoU(float const * const a, float const * const b) { float left = max(a[0], b[0]), right = min(a[2], b[2]); float top = max(a[1], b[1]), bottom = min(a[3], b[3]); float width = max(right - left + 1, 0.f), height = max(bottom - top + 1, 0.f); float interS = width * height; float Sa = (a[2] - a[0] + 1) * (a[3] - a[1] + 1); float Sb = (b[2] - b[0] + 1) * (b[3] - b[1] + 1); return interS / (Sa + Sb - interS); } __global__ void nms_kernel(const int n_boxes, const float nms_overlap_thresh, const float *dev_boxes, unsigned long long *dev_mask) { const int row_start = blockIdx.y; const int col_start = blockIdx.x; // if (row_start > col_start) return; const int row_size = min(n_boxes - row_start * threadsPerBlock, threadsPerBlock); const int col_size = min(n_boxes - col_start * threadsPerBlock, threadsPerBlock); __shared__ float block_boxes[threadsPerBlock * 5]; if (threadIdx.x < col_size) { block_boxes[threadIdx.x * 5 + 0] = dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 0]; block_boxes[threadIdx.x * 5 + 1] = dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 1]; block_boxes[threadIdx.x * 5 + 2] = dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 2]; block_boxes[threadIdx.x * 5 + 3] = dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 3]; block_boxes[threadIdx.x * 5 + 4] = dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 4]; } __syncthreads(); if (threadIdx.x < row_size) { const int cur_box_idx = threadsPerBlock * row_start + threadIdx.x; const float *cur_box = dev_boxes + cur_box_idx * 5; int i = 0; unsigned long long t = 0; int start = 0; if (row_start == col_start) { start = threadIdx.x + 1; } for (i = start; i < col_size; i++) { if (devIoU(cur_box, block_boxes + i * 5) > nms_overlap_thresh) { t |= 1ULL << i; } } const int col_blocks = DIVUP(n_boxes, threadsPerBlock); dev_mask[cur_box_idx * col_blocks + col_start] = t; } } void _set_device(int device_id) { int current_device; CUDA_CHECK(cudaGetDevice(¤t_device)); if (current_device == device_id) { return; } // The call to cudaSetDevice must come before any calls to Get, which // may perform initialization using the GPU. CUDA_CHECK(cudaSetDevice(device_id)); } void _nms(int* keep_out, int* num_out, const float* boxes_host, int boxes_num, int boxes_dim, float nms_overlap_thresh, int device_id) { _set_device(device_id); float* boxes_dev = NULL; unsigned long long* mask_dev = NULL; const int col_blocks = DIVUP(boxes_num, threadsPerBlock); CUDA_CHECK(cudaMalloc(&boxes_dev, boxes_num * boxes_dim * sizeof(float))); CUDA_CHECK(cudaMemcpy(boxes_dev, boxes_host, boxes_num * boxes_dim * sizeof(float), cudaMemcpyHostToDevice)); CUDA_CHECK(cudaMalloc(&mask_dev, boxes_num * col_blocks * sizeof(unsigned long long))); dim3 blocks(DIVUP(boxes_num, threadsPerBlock), DIVUP(boxes_num, threadsPerBlock)); dim3 threads(threadsPerBlock); nms_kernel<<>>(boxes_num, nms_overlap_thresh, boxes_dev, mask_dev); std::vector mask_host(boxes_num * col_blocks); CUDA_CHECK(cudaMemcpy(&mask_host[0], mask_dev, sizeof(unsigned long long) * boxes_num * col_blocks, cudaMemcpyDeviceToHost)); std::vector remv(col_blocks); memset(&remv[0], 0, sizeof(unsigned long long) * col_blocks); int num_to_keep = 0; for (int i = 0; i < boxes_num; i++) { int nblock = i / threadsPerBlock; int inblock = i % threadsPerBlock; if (!(remv[nblock] & (1ULL << inblock))) { keep_out[num_to_keep++] = i; unsigned long long *p = &mask_host[0] + i * col_blocks; for (int j = nblock; j < col_blocks; j++) { remv[j] |= p[j]; } } } *num_out = num_to_keep; CUDA_CHECK(cudaFree(boxes_dev)); CUDA_CHECK(cudaFree(mask_dev)); } ================================================ FILE: lib/nms/setup_linux.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong # -------------------------------------------------------- # Based on: # py-faster-rcnn # Copyright (c) 2015 Microsoft # Licence under The MIT License # https://github.com/rbgirshick/py-faster-rcnn # -------------------------------------------------------- import os from os.path import join as pjoin from setuptools import setup from distutils.extension import Extension from Cython.Distutils import build_ext import numpy as np def find_in_path(name, path): "Find a file in a search path" # Adapted fom # http://code.activestate.com/recipes/52224-find-a-file-given-a-search-path/ for dir in path.split(os.pathsep): binpath = pjoin(dir, name) if os.path.exists(binpath): return os.path.abspath(binpath) return None def locate_cuda(): """Locate the CUDA environment on the system Returns a dict with keys 'home', 'nvcc', 'include', and 'lib64' and values giving the absolute path to each directory. Starts by looking for the CUDAHOME env variable. If not found, everything is based on finding 'nvcc' in the PATH. """ # first check if the CUDAHOME env variable is in use if 'CUDAHOME' in os.environ: home = os.environ['CUDAHOME'] nvcc = pjoin(home, 'bin', 'nvcc') else: # otherwise, search the PATH for NVCC default_path = pjoin(os.sep, 'usr', 'local', 'cuda', 'bin') nvcc = find_in_path('nvcc', os.environ['PATH'] + os.pathsep + default_path) if nvcc is None: raise EnvironmentError('The nvcc binary could not be ' 'located in your $PATH. Either add it to your path, or set $CUDAHOME') home = os.path.dirname(os.path.dirname(nvcc)) cudaconfig = {'home':home, 'nvcc':nvcc, 'include': pjoin(home, 'include'), 'lib64': pjoin(home, 'lib64')} for k, v in cudaconfig.iteritems(): if not os.path.exists(v): raise EnvironmentError('The CUDA %s path could not be located in %s' % (k, v)) return cudaconfig CUDA = locate_cuda() # Obtain the numpy include directory. This logic works across numpy versions. try: numpy_include = np.get_include() except AttributeError: numpy_include = np.get_numpy_include() def customize_compiler_for_nvcc(self): """inject deep into distutils to customize how the dispatch to gcc/nvcc works. If you subclass UnixCCompiler, it's not trivial to get your subclass injected in, and still have the right customizations (i.e. distutils.sysconfig.customize_compiler) run on it. So instead of going the OO route, I have this. Note, it's kindof like a wierd functional subclassing going on.""" # tell the compiler it can processes .cu self.src_extensions.append('.cu') # save references to the default compiler_so and _comple methods default_compiler_so = self.compiler_so super = self._compile # now redefine the _compile method. This gets executed for each # object but distutils doesn't have the ability to change compilers # based on source extension: we add it. def _compile(obj, src, ext, cc_args, extra_postargs, pp_opts): if os.path.splitext(src)[1] == '.cu': # use the cuda for .cu files self.set_executable('compiler_so', CUDA['nvcc']) # use only a subset of the extra_postargs, which are 1-1 translated # from the extra_compile_args in the Extension class postargs = extra_postargs['nvcc'] else: postargs = extra_postargs['gcc'] super(obj, src, ext, cc_args, postargs, pp_opts) # reset the default compiler_so, which we might have changed for cuda self.compiler_so = default_compiler_so # inject our redefined _compile method into the class self._compile = _compile # run the customize_compiler class custom_build_ext(build_ext): def build_extensions(self): customize_compiler_for_nvcc(self.compiler) build_ext.build_extensions(self) ext_modules = [ Extension( "cpu_nms", ["cpu_nms.pyx"], extra_compile_args={'gcc': ["-Wno-cpp", "-Wno-unused-function"]}, include_dirs = [numpy_include] ), Extension('gpu_nms', ['nms_kernel.cu', 'gpu_nms.pyx'], library_dirs=[CUDA['lib64']], libraries=['cudart'], language='c++', runtime_library_dirs=[CUDA['lib64']], # this syntax is specific to this build system # we're only going to use certain compiler args with nvcc and not with # gcc the implementation of this trick is in customize_compiler() below extra_compile_args={'gcc': ["-Wno-unused-function"], 'nvcc': ['-arch=sm_35', '--ptxas-options=-v', '-c', '--compiler-options', "'-fPIC'"]}, include_dirs = [numpy_include, CUDA['include']] ), ] setup( name='nms', ext_modules=ext_modules, # inject our custom trigger cmdclass={'build_ext': custom_build_ext}, ) ================================================ FILE: lib/nms/setup_windows.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong # -------------------------------------------------------- # Based on: # py-faster-rcnn # Copyright (c) 2015 Microsoft # Licence under The MIT License # https://github.com/rbgirshick/py-faster-rcnn # -------------------------------------------------------- import numpy as np import os from os.path import join as pjoin #from distutils.core import setup from setuptools import setup from distutils.extension import Extension from Cython.Distutils import build_ext import subprocess #change for windows, by MrX nvcc_bin = 'nvcc.exe' lib_dir = 'lib/x64' import distutils.msvc9compiler distutils.msvc9compiler.VERSION = 14.0 def find_in_path(name, path): "Find a file in a search path" # Adapted fom # http://code.activestate.com/recipes/52224-find-a-file-given-a-search-path/ for dir in path.split(os.pathsep): binpath = pjoin(dir, name) if os.path.exists(binpath): return os.path.abspath(binpath) return None def locate_cuda(): """Locate the CUDA environment on the system Returns a dict with keys 'home', 'nvcc', 'include', and 'lib64' and values giving the absolute path to each directory. Starts by looking for the CUDAHOME env variable. If not found, everything is based on finding 'nvcc' in the PATH. """ # first check if the CUDAHOME env variable is in use if 'CUDA_PATH' in os.environ: home = os.environ['CUDA_PATH'] print("home = %s\n" % home) nvcc = pjoin(home, 'bin', nvcc_bin) else: # otherwise, search the PATH for NVCC default_path = pjoin(os.sep, 'usr', 'local', 'cuda', 'bin') nvcc = find_in_path(nvcc_bin, os.environ['PATH'] + os.pathsep + default_path) if nvcc is None: raise EnvironmentError('The nvcc binary could not be ' 'located in your $PATH. Either add it to your path, or set $CUDA_PATH') home = os.path.dirname(os.path.dirname(nvcc)) print("home = %s, nvcc = %s\n" % (home, nvcc)) cudaconfig = {'home':home, 'nvcc':nvcc, 'include': pjoin(home, 'include'), 'lib64': pjoin(home, lib_dir)} for k, v in cudaconfig.iteritems(): if not os.path.exists(v): raise EnvironmentError('The CUDA %s path could not be located in %s' % (k, v)) return cudaconfig CUDA = locate_cuda() # Obtain the numpy include directory. This logic works across numpy versions. try: numpy_include = np.get_include() except AttributeError: numpy_include = np.get_numpy_include() def customize_compiler_for_nvcc(self): """inject deep into distutils to customize how the dispatch to gcc/nvcc works. If you subclass UnixCCompiler, it's not trivial to get your subclass injected in, and still have the right customizations (i.e. distutils.sysconfig.customize_compiler) run on it. So instead of going the OO route, I have this. Note, it's kindof like a wierd functional subclassing going on.""" # tell the compiler it can processes .cu #self.src_extensions.append('.cu') # save references to the default compiler_so and _comple methods #default_compiler_so = self.spawn #default_compiler_so = self.rc super = self.compile # now redefine the _compile method. This gets executed for each # object but distutils doesn't have the ability to change compilers # based on source extension: we add it. def compile(sources, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None): postfix=os.path.splitext(sources[0])[1] if postfix == '.cu': # use the cuda for .cu files #self.set_executable('compiler_so', CUDA['nvcc']) # use only a subset of the extra_postargs, which are 1-1 translated # from the extra_compile_args in the Extension class postargs = extra_postargs['nvcc'] else: postargs = extra_postargs['gcc'] return super(sources, output_dir, macros, include_dirs, debug, extra_preargs, postargs, depends) # reset the default compiler_so, which we might have changed for cuda #self.rc = default_compiler_so # inject our redefined _compile method into the class self.compile = compile # run the customize_compiler class custom_build_ext(build_ext): def build_extensions(self): customize_compiler_for_nvcc(self.compiler) build_ext.build_extensions(self) ext_modules = [ # unix _compile: obj, src, ext, cc_args, extra_postargs, pp_opts Extension( "cpu_nms", sources=["cpu_nms.pyx"], extra_compile_args={'gcc': []}, include_dirs = [numpy_include], ), ] setup( name='fast_rcnn', ext_modules=ext_modules, # inject our custom trigger cmdclass={'build_ext': custom_build_ext}, ) ================================================ FILE: lib/nms/setup_windows_cuda.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong # -------------------------------------------------------- #!/usr/bin/env python import numpy as np import os # on Windows, we need the original PATH without Anaconda's compiler in it: PATH = os.environ.get('PATH') + ';C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin' from distutils.spawn import spawn, find_executable from setuptools import setup, find_packages, Extension from setuptools.command.build_ext import build_ext import sys # CUDA specific config # nvcc is assumed to be in user's PATH nvcc_compile_args = ['-O', '--ptxas-options=-v', '-arch=compute_35', '-code=sm_35,sm_52,sm_61', '-c', '--compiler-options=-fPIC'] nvcc_compile_args = os.environ.get('NVCCFLAGS', '').split() + nvcc_compile_args cuda_libs = ['cublas'] nvcc_bin = 'nvcc.exe' lib_dir = 'lib/x64' import distutils.msvc9compiler distutils.msvc9compiler.VERSION = 14.0 # Obtain the numpy include directory. This logic works across numpy versions. try: numpy_include = np.get_include() except AttributeError: numpy_include = np.get_numpy_include() cudamat_ext = Extension('gpu_nms', sources=[ 'gpu_nms.cu' ], language='c++', libraries=cuda_libs, extra_compile_args=nvcc_compile_args, include_dirs = [numpy_include, 'C:\\Programming\\CUDA\\v8.0\\include']) class CUDA_build_ext(build_ext): """ Custom build_ext command that compiles CUDA files. Note that all extension source files will be processed with this compiler. """ def build_extensions(self): self.compiler.src_extensions.append('.cu') self.compiler.set_executable('compiler_so', 'nvcc') self.compiler.set_executable('linker_so', 'nvcc --shared') if hasattr(self.compiler, '_c_extensions'): self.compiler._c_extensions.append('.cu') # needed for Windows self.compiler.spawn = self.spawn build_ext.build_extensions(self) def spawn(self, cmd, search_path=1, verbose=0, dry_run=0): """ Perform any CUDA specific customizations before actually launching compile/link etc. commands. """ if (sys.platform == 'darwin' and len(cmd) >= 2 and cmd[0] == 'nvcc' and cmd[1] == '--shared' and cmd.count('-arch') > 0): # Versions of distutils on OSX earlier than 2.7.9 inject # '-arch x86_64' which we need to strip while using nvcc for # linking while True: try: index = cmd.index('-arch') del cmd[index:index+2] except ValueError: break elif self.compiler.compiler_type == 'msvc': # There are several things we need to do to change the commands # issued by MSVCCompiler into one that works with nvcc. In the end, # it might have been easier to write our own CCompiler class for # nvcc, as we're only interested in creating a shared library to # load with ctypes, not in creating an importable Python extension. # - First, we replace the cl.exe or link.exe call with an nvcc # call. In case we're running Anaconda, we search cl.exe in the # original search path we captured further above -- Anaconda # inserts a MSVC version into PATH that is too old for nvcc. cmd[:1] = ['nvcc', '--compiler-bindir', os.path.dirname(find_executable("cl.exe", PATH)) or cmd[0]] # - Secondly, we fix a bunch of command line arguments. for idx, c in enumerate(cmd): # create .dll instead of .pyd files #if '.pyd' in c: cmd[idx] = c = c.replace('.pyd', '.dll') #20160601, by MrX # replace /c by -c if c == '/c': cmd[idx] = '-c' # replace /DLL by --shared elif c == '/DLL': cmd[idx] = '--shared' # remove --compiler-options=-fPIC elif '-fPIC' in c: del cmd[idx] # replace /Tc... by ... elif c.startswith('/Tc'): cmd[idx] = c[3:] # replace /Fo... by -o ... elif c.startswith('/Fo'): cmd[idx:idx+1] = ['-o', c[3:]] # replace /LIBPATH:... by -L... elif c.startswith('/LIBPATH:'): cmd[idx] = '-L' + c[9:] # replace /OUT:... by -o ... elif c.startswith('/OUT:'): cmd[idx:idx+1] = ['-o', c[5:]] # remove /EXPORT:initlibcudamat or /EXPORT:initlibcudalearn elif c.startswith('/EXPORT:'): del cmd[idx] # replace cublas.lib by -lcublas elif c == 'cublas.lib': cmd[idx] = '-lcublas' # - Finally, we pass on all arguments starting with a '/' to the # compiler or linker, and have nvcc handle all other arguments if '--shared' in cmd: pass_on = '--linker-options=' # we only need MSVCRT for a .dll, remove CMT if it sneaks in: cmd.append('/NODEFAULTLIB:libcmt.lib') else: pass_on = '--compiler-options=' cmd = ([c for c in cmd if c[0] != '/'] + [pass_on + ','.join(c for c in cmd if c[0] == '/')]) # For the future: Apart from the wrongly set PATH by Anaconda, it # would suffice to run the following for compilation on Windows: # nvcc -c -O -o .obj .cu # And the following for linking: # nvcc --shared -o .dll .obj .obj -lcublas # This could be done by a NVCCCompiler class for all platforms. spawn(cmd, search_path, verbose, dry_run) setup(name="py_fast_rcnn_gpu", description="Performs linear algebra computation on the GPU via CUDA", ext_modules=[cudamat_ext], cmdclass={'build_ext': CUDA_build_ext}, ) ================================================ FILE: lib/rpn/__init__.py ================================================ ================================================ FILE: lib/rpn/generate_anchor.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- """ Generate base anchors on index 0 """ import numpy as np def generate_anchors(base_size=16, ratios=[0.5, 1, 2], scales=2 ** np.arange(3, 6)): """ Generate anchor (reference) windows by enumerating aspect ratios X scales wrt a reference (0, 0, 15, 15) window. """ base_anchor = np.array([1, 1, base_size, base_size]) - 1 ratio_anchors = _ratio_enum(base_anchor, ratios) anchors = np.vstack([_scale_enum(ratio_anchors[i, :], scales) for i in xrange(ratio_anchors.shape[0])]) return anchors def _whctrs(anchor): """ Return width, height, x center, and y center for an anchor (window). """ w = anchor[2] - anchor[0] + 1 h = anchor[3] - anchor[1] + 1 x_ctr = anchor[0] + 0.5 * (w - 1) y_ctr = anchor[1] + 0.5 * (h - 1) return w, h, x_ctr, y_ctr def _mkanchors(ws, hs, x_ctr, y_ctr): """ Given a vector of widths (ws) and heights (hs) around a center (x_ctr, y_ctr), output a set of anchors (windows). """ ws = ws[:, np.newaxis] hs = hs[:, np.newaxis] anchors = np.hstack((x_ctr - 0.5 * (ws - 1), y_ctr - 0.5 * (hs - 1), x_ctr + 0.5 * (ws - 1), y_ctr + 0.5 * (hs - 1))) return anchors def _ratio_enum(anchor, ratios): """ Enumerate a set of anchors for each aspect ratio wrt an anchor. """ w, h, x_ctr, y_ctr = _whctrs(anchor) size = w * h size_ratios = size / ratios ws = np.round(np.sqrt(size_ratios)) hs = np.round(ws * ratios) anchors = _mkanchors(ws, hs, x_ctr, y_ctr) return anchors def _scale_enum(anchor, scales): """ Enumerate a set of anchors for each scale wrt an anchor. """ w, h, x_ctr, y_ctr = _whctrs(anchor) ws = w * scales hs = h * scales anchors = _mkanchors(ws, hs, x_ctr, y_ctr) return anchors ================================================ FILE: lib/rpn/rpn.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Haozhi Qi # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- """ RPN: data = {'data': [num_images, c, h, w], 'im_info': [num_images, 4] (optional)} label = {'gt_boxes': [num_boxes, 5] (optional), 'label': [batch_size, 1] <- [batch_size, num_anchors, feat_height, feat_width], 'bbox_target': [batch_size, num_anchors, feat_height, feat_width], 'bbox_weight': [batch_size, num_anchors, feat_height, feat_width]} """ import numpy as np import numpy.random as npr from utils.image import get_image, get_pair_image, tensor_vstack from generate_anchor import generate_anchors from bbox.bbox_transform import bbox_overlaps, bbox_transform def get_rpn_testbatch(roidb, cfg): """ return a dict of testbatch :param roidb: ['image', 'flipped'] :return: data, label, im_info """ # assert len(roidb) == 1, 'Single batch only' imgs, roidb = get_image(roidb, cfg) im_array = imgs im_info = [np.array([roidb[i]['im_info']], dtype=np.float32) for i in range(len(roidb))] data = [{'data': im_array[i], 'im_info': im_info[i]} for i in range(len(roidb))] label = {} return data, label, im_info def get_rpn_batch(roidb, cfg): """ prototype for rpn batch: data, im_info, gt_boxes :param roidb: ['image', 'flipped'] + ['gt_boxes', 'boxes', 'gt_classes'] :return: data, label """ assert len(roidb) == 1, 'Single batch only' imgs, roidb = get_image(roidb, cfg) im_array = imgs[0] im_info = np.array([roidb[0]['im_info']], dtype=np.float32) # gt boxes: (x1, y1, x2, y2, cls) if roidb[0]['gt_classes'].size > 0: gt_inds = np.where(roidb[0]['gt_classes'] != 0)[0] gt_boxes = np.empty((roidb[0]['boxes'].shape[0], 5), dtype=np.float32) gt_boxes[:, 0:4] = roidb[0]['boxes'][gt_inds, :] gt_boxes[:, 4] = roidb[0]['gt_classes'][gt_inds] else: gt_boxes = np.empty((0, 5), dtype=np.float32) data = {'data': im_array, 'im_info': im_info} label = {'gt_boxes': gt_boxes} return data, label def get_rpn_pair_batch(roidb, cfg): """ prototype for rpn batch: data, im_info, gt_boxes :param roidb: ['image', 'flipped'] + ['gt_boxes', 'boxes', 'gt_classes'] :return: data, label """ assert len(roidb) == 1, 'Single batch only' imgs, ref_imgs, eq_flags, roidb = get_pair_image(roidb, cfg) im_array = imgs[0] ref_im_array = ref_imgs[0] eq_flag_array = np.array([eq_flags[0],], dtype=np.float32) im_info = np.array([roidb[0]['im_info']], dtype=np.float32) # gt boxes: (x1, y1, x2, y2, cls) if roidb[0]['gt_classes'].size > 0: gt_inds = np.where(roidb[0]['gt_classes'] != 0)[0] gt_boxes = np.empty((roidb[0]['boxes'].shape[0], 5), dtype=np.float32) gt_boxes[:, 0:4] = roidb[0]['boxes'][gt_inds, :] gt_boxes[:, 4] = roidb[0]['gt_classes'][gt_inds] else: gt_boxes = np.empty((0, 5), dtype=np.float32) data = {'data': im_array, 'data_ref': ref_im_array, 'eq_flag': eq_flag_array, 'im_info': im_info} label = {'gt_boxes': gt_boxes} return data, label def assign_anchor(feat_shape, gt_boxes, im_info, cfg, feat_stride=16, scales=(8, 16, 32), ratios=(0.5, 1, 2), allowed_border=0, normalize_target=False, bbox_mean=(0.0, 0.0, 0.0, 0.0), bbox_std=(0.1, 0.1, 0.4, 0.4)): """ assign ground truth boxes to anchor positions :param feat_shape: infer output shape :param gt_boxes: assign ground truth :param im_info: filter out anchors overlapped with edges :param feat_stride: anchor position step :param scales: used to generate anchors, affects num_anchors (per location) :param ratios: aspect ratios of generated anchors :param allowed_border: filter out anchors with edge overlap > allowed_border :param normalize_target: normalize rpn target :param bbox_mean: anchor target mean :param bbox_std: anchor target std :return: dict of label 'label': of shape (batch_size, 1) <- (batch_size, num_anchors, feat_height, feat_width) 'bbox_target': of shape (batch_size, num_anchors * 4, feat_height, feat_width) 'bbox_inside_weight': *todo* mark the assigned anchors 'bbox_outside_weight': used to normalize the bbox_loss, all weights sums to RPN_POSITIVE_WEIGHT """ def _unmap(data, count, inds, fill=0): """" unmap a subset inds of data into original data of size count """ if len(data.shape) == 1: ret = np.empty((count,), dtype=np.float32) ret.fill(fill) ret[inds] = data else: ret = np.empty((count,) + data.shape[1:], dtype=np.float32) ret.fill(fill) ret[inds, :] = data return ret DEBUG = False im_info = im_info[0] scales = np.array(scales, dtype=np.float32) base_anchors = generate_anchors(base_size=feat_stride, ratios=list(ratios), scales=scales) num_anchors = base_anchors.shape[0] feat_height, feat_width = feat_shape[-2:] if DEBUG: print 'anchors:' print base_anchors print 'anchor shapes:' print np.hstack((base_anchors[:, 2::4] - base_anchors[:, 0::4], base_anchors[:, 3::4] - base_anchors[:, 1::4])) print 'im_info', im_info print 'height', feat_height, 'width', feat_width print 'gt_boxes shape', gt_boxes.shape print 'gt_boxes', gt_boxes # 1. generate proposals from bbox deltas and shifted anchors shift_x = np.arange(0, feat_width) * feat_stride shift_y = np.arange(0, feat_height) * feat_stride shift_x, shift_y = np.meshgrid(shift_x, shift_y) shifts = np.vstack((shift_x.ravel(), shift_y.ravel(), shift_x.ravel(), shift_y.ravel())).transpose() # add A anchors (1, A, 4) to # cell K shifts (K, 1, 4) to get # shift anchors (K, A, 4) # reshape to (K*A, 4) shifted anchors A = num_anchors K = shifts.shape[0] all_anchors = base_anchors.reshape((1, A, 4)) + shifts.reshape((1, K, 4)).transpose((1, 0, 2)) all_anchors = all_anchors.reshape((K * A, 4)) total_anchors = int(K * A) # only keep anchors inside the image inds_inside = np.where((all_anchors[:, 0] >= -allowed_border) & (all_anchors[:, 1] >= -allowed_border) & (all_anchors[:, 2] < im_info[1] + allowed_border) & (all_anchors[:, 3] < im_info[0] + allowed_border))[0] if DEBUG: print 'total_anchors', total_anchors print 'inds_inside', len(inds_inside) # keep only inside anchors anchors = all_anchors[inds_inside, :] if DEBUG: print 'anchors shape', anchors.shape # label: 1 is positive, 0 is negative, -1 is dont care labels = np.empty((len(inds_inside),), dtype=np.float32) labels.fill(-1) if gt_boxes.size > 0: # overlap between the anchors and the gt boxes # overlaps (ex, gt) overlaps = bbox_overlaps(anchors.astype(np.float), gt_boxes.astype(np.float)) argmax_overlaps = overlaps.argmax(axis=1) max_overlaps = overlaps[np.arange(len(inds_inside)), argmax_overlaps] gt_argmax_overlaps = overlaps.argmax(axis=0) gt_max_overlaps = overlaps[gt_argmax_overlaps, np.arange(overlaps.shape[1])] gt_argmax_overlaps = np.where(overlaps == gt_max_overlaps)[0] if not cfg.TRAIN.RPN_CLOBBER_POSITIVES: # assign bg labels first so that positive labels can clobber them labels[max_overlaps < cfg.TRAIN.RPN_NEGATIVE_OVERLAP] = 0 # fg label: for each gt, anchor with highest overlap labels[gt_argmax_overlaps] = 1 # fg label: above threshold IoU labels[max_overlaps >= cfg.TRAIN.RPN_POSITIVE_OVERLAP] = 1 if cfg.TRAIN.RPN_CLOBBER_POSITIVES: # assign bg labels last so that negative labels can clobber positives labels[max_overlaps < cfg.TRAIN.RPN_NEGATIVE_OVERLAP] = 0 else: labels[:] = 0 # subsample positive labels if we have too many num_fg = int(cfg.TRAIN.RPN_FG_FRACTION * cfg.TRAIN.RPN_BATCH_SIZE) fg_inds = np.where(labels == 1)[0] if len(fg_inds) > num_fg: disable_inds = npr.choice(fg_inds, size=(len(fg_inds) - num_fg), replace=False) if DEBUG: disable_inds = fg_inds[:(len(fg_inds) - num_fg)] labels[disable_inds] = -1 # subsample negative labels if we have too many num_bg = cfg.TRAIN.RPN_BATCH_SIZE - np.sum(labels == 1) bg_inds = np.where(labels == 0)[0] if len(bg_inds) > num_bg: disable_inds = npr.choice(bg_inds, size=(len(bg_inds) - num_bg), replace=False) if DEBUG: disable_inds = bg_inds[:(len(bg_inds) - num_bg)] labels[disable_inds] = -1 bbox_targets = np.zeros((len(inds_inside), 4), dtype=np.float32) if gt_boxes.size > 0: bbox_targets[:] = bbox_transform(anchors, gt_boxes[argmax_overlaps, :4]) bbox_weights = np.zeros((len(inds_inside), 4), dtype=np.float32) bbox_weights[labels == 1, :] = np.array(cfg.TRAIN.RPN_BBOX_WEIGHTS) if DEBUG: _sums = bbox_targets[labels == 1, :].sum(axis=0) _squared_sums = (bbox_targets[labels == 1, :] ** 2).sum(axis=0) _counts = np.sum(labels == 1) means = _sums / (_counts + 1e-14) stds = np.sqrt(_squared_sums / _counts - means ** 2) print 'means', means print 'stdevs', stds if normalize_target: bbox_targets = ((bbox_targets - np.array(bbox_mean)) / np.array(bbox_std)) # map up to original set of anchors labels = _unmap(labels, total_anchors, inds_inside, fill=-1) bbox_targets = _unmap(bbox_targets, total_anchors, inds_inside, fill=0) bbox_weights = _unmap(bbox_weights, total_anchors, inds_inside, fill=0) if DEBUG: print 'rpn: max max_overlaps', np.max(max_overlaps) print 'rpn: num_positives', np.sum(labels == 1) print 'rpn: num_negatives', np.sum(labels == 0) _fg_sum = np.sum(labels == 1) _bg_sum = np.sum(labels == 0) _count = 1 print 'rpn: num_positive avg', _fg_sum / _count print 'rpn: num_negative avg', _bg_sum / _count labels = labels.reshape((1, feat_height, feat_width, A)).transpose(0, 3, 1, 2) labels = labels.reshape((1, A * feat_height * feat_width)) bbox_targets = bbox_targets.reshape((1, feat_height, feat_width, A * 4)).transpose(0, 3, 1, 2) bbox_weights = bbox_weights.reshape((1, feat_height, feat_width, A * 4)).transpose((0, 3, 1, 2)) label = {'label': labels, 'bbox_target': bbox_targets, 'bbox_weight': bbox_weights} return label ================================================ FILE: lib/utils/PrefetchingIter.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- import mxnet as mx from mxnet.io import DataDesc, DataBatch import threading class PrefetchingIter(mx.io.DataIter): """Base class for prefetching iterators. Takes one or more DataIters ( or any class with "reset" and "next" methods) and combine them with prefetching. For example: Parameters ---------- iters : DataIter or list of DataIter one or more DataIters (or any class with "reset" and "next" methods) rename_data : None or list of dict i-th element is a renaming map for i-th iter, in the form of {'original_name' : 'new_name'}. Should have one entry for each entry in iter[i].provide_data rename_label : None or list of dict Similar to rename_data Examples -------- iter = PrefetchingIter([NDArrayIter({'data': X1}), NDArrayIter({'data': X2})], rename_data=[{'data': 'data1'}, {'data': 'data2'}]) """ def __init__(self, iters, rename_data=None, rename_label=None): super(PrefetchingIter, self).__init__() if not isinstance(iters, list): iters = [iters] self.n_iter = len(iters) assert self.n_iter ==1, "Our prefetching iter only support 1 DataIter" self.iters = iters self.rename_data = rename_data self.rename_label = rename_label self.batch_size = len(self.provide_data) * self.provide_data[0][0][1][0] self.data_ready = [threading.Event() for i in range(self.n_iter)] self.data_taken = [threading.Event() for i in range(self.n_iter)] for e in self.data_taken: e.set() self.started = True self.current_batch = [None for _ in range(self.n_iter)] self.next_batch = [None for _ in range(self.n_iter)] def prefetch_func(self, i): """Thread entry""" while True: self.data_taken[i].wait() if not self.started: break try: self.next_batch[i] = self.iters[i].next() except StopIteration: self.next_batch[i] = None self.data_taken[i].clear() self.data_ready[i].set() self.prefetch_threads = [threading.Thread(target=prefetch_func, args=[self, i]) \ for i in range(self.n_iter)] for thread in self.prefetch_threads: thread.setDaemon(True) thread.start() def __del__(self): self.started = False for e in self.data_taken: e.set() for thread in self.prefetch_threads: thread.join() @property def provide_data(self): """The name and shape of data provided by this iterator""" if self.rename_data is None: return sum([i.provide_data for i in self.iters], []) else: return sum([[ DataDesc(r[x.name], x.shape, x.dtype) if isinstance(x, DataDesc) else DataDesc(*x) for x in i.provide_data ] for r, i in zip(self.rename_data, self.iters)], []) @property def provide_label(self): """The name and shape of label provided by this iterator""" if self.rename_label is None: return sum([i.provide_label for i in self.iters], []) else: return sum([[ DataDesc(r[x.name], x.shape, x.dtype) if isinstance(x, DataDesc) else DataDesc(*x) for x in i.provide_label ] for r, i in zip(self.rename_label, self.iters)], []) def reset(self): for e in self.data_ready: e.wait() for i in self.iters: i.reset() for e in self.data_ready: e.clear() for e in self.data_taken: e.set() def iter_next(self): for e in self.data_ready: e.wait() if self.next_batch[0] is None: return False else: self.current_batch = self.next_batch[0] for e in self.data_ready: e.clear() for e in self.data_taken: e.set() return True def next(self): if self.iter_next(): return self.current_batch else: raise StopIteration def getdata(self): return self.current_batch.data def getlabel(self): return self.current_batch.label def getindex(self): return self.current_batch.index def getpad(self): return self.current_batch.pad ================================================ FILE: lib/utils/__init__.py ================================================ ================================================ FILE: lib/utils/combine_model.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong # -------------------------------------------------------- from load_model import load_checkpoint from save_model import save_checkpoint def combine_model(prefix1, epoch1, prefix2, epoch2, prefix_out, epoch_out): args1, auxs1 = load_checkpoint(prefix1, epoch1) args2, auxs2 = load_checkpoint(prefix2, epoch2) arg_names = args1.keys() + args2.keys() aux_names = auxs1.keys() + auxs2.keys() args = dict() for arg in arg_names: if arg in args1: args[arg] = args1[arg] if arg in args2: args[arg] = args2[arg] auxs = dict() for aux in aux_names: if aux in auxs1: auxs[aux] = auxs1[aux] if aux in auxs2: auxs[aux] = auxs2[aux] save_checkpoint(prefix_out, epoch_out, args, auxs) ================================================ FILE: lib/utils/create_logger.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Bin Xiao # -------------------------------------------------------- import os import logging import time def create_logger(root_output_path, cfg, image_set): # set up logger if not os.path.exists(root_output_path): os.makedirs(root_output_path) assert os.path.exists(root_output_path), '{} does not exist'.format(root_output_path) cfg_name = os.path.basename(cfg).split('.')[0] config_output_path = os.path.join(root_output_path, '{}'.format(cfg_name)) if not os.path.exists(config_output_path): os.makedirs(config_output_path) image_sets = [iset for iset in image_set.split('+')] final_output_path = os.path.join(config_output_path, '{}'.format('_'.join(image_sets))) if not os.path.exists(final_output_path): os.makedirs(final_output_path) log_file = '{}_{}.log'.format(cfg_name, time.strftime('%Y-%m-%d-%H-%M')) head = '%(asctime)-15s %(message)s' logging.basicConfig(filename=os.path.join(final_output_path, log_file), format=head) logger = logging.getLogger() logger.setLevel(logging.INFO) return logger, final_output_path ================================================ FILE: lib/utils/image.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong # -------------------------------------------------------- import numpy as np import os import cv2 import random from PIL import Image from bbox.bbox_transform import clip_boxes # TODO: This two functions should be merged with individual data loader def get_image(roidb, config): """ preprocess image and return processed roidb :param roidb: a list of roidb :return: list of img as in mxnet format roidb add new item['im_info'] 0 --- x (width, second dim of im) | y (height, first dim of im) """ num_images = len(roidb) processed_ims = [] processed_roidb = [] for i in range(num_images): roi_rec = roidb[i] assert os.path.exists(roi_rec['image']), '%s does not exist'.format(roi_rec['image']) im = cv2.imread(roi_rec['image'], cv2.IMREAD_COLOR|cv2.IMREAD_IGNORE_ORIENTATION) if roidb[i]['flipped']: im = im[:, ::-1, :] new_rec = roi_rec.copy() scale_ind = random.randrange(len(config.SCALES)) target_size = config.SCALES[scale_ind][0] max_size = config.SCALES[scale_ind][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) processed_ims.append(im_tensor) im_info = [im_tensor.shape[2], im_tensor.shape[3], im_scale] new_rec['boxes'] = clip_boxes(np.round(roi_rec['boxes'].copy() * im_scale), im_info[:2]) new_rec['im_info'] = im_info processed_roidb.append(new_rec) return processed_ims, processed_roidb def get_pair_image(roidb, config): """ preprocess image and return processed roidb :param roidb: a list of roidb :return: list of img as in mxnet format roidb add new item['im_info'] 0 --- x (width, second dim of im) | y (height, first dim of im) """ num_images = len(roidb) processed_ims = [] processed_ref_ims = [] processed_eq_flags = [] processed_roidb = [] for i in range(num_images): roi_rec = roidb[i] eq_flag = 0 # 0 for unequal, 1 for equal assert os.path.exists(roi_rec['image']), '%s does not exist'.format(roi_rec['image']) im = cv2.imread(roi_rec['image'], cv2.IMREAD_COLOR|cv2.IMREAD_IGNORE_ORIENTATION) if roi_rec.has_key('pattern'): ref_id = min(max(roi_rec['frame_seg_id'] + np.random.randint(config.TRAIN.MIN_OFFSET, config.TRAIN.MAX_OFFSET+1), 0),roi_rec['frame_seg_len']-1) ref_image = roi_rec['pattern'] % ref_id assert os.path.exists(ref_image), '%s does not exist'.format(ref_image) ref_im = cv2.imread(ref_image, cv2.IMREAD_COLOR|cv2.IMREAD_IGNORE_ORIENTATION) if ref_id == roi_rec['frame_seg_id']: eq_flag = 1 else: ref_im = im.copy() eq_flag = 1 if roidb[i]['flipped']: im = im[:, ::-1, :] ref_im = ref_im[:, ::-1, :] new_rec = roi_rec.copy() scale_ind = random.randrange(len(config.SCALES)) target_size = config.SCALES[scale_ind][0] max_size = config.SCALES[scale_ind][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) ref_im, im_scale = resize(ref_im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) ref_im_tensor = transform(ref_im, config.network.PIXEL_MEANS) processed_ims.append(im_tensor) processed_ref_ims.append(ref_im_tensor) processed_eq_flags.append(eq_flag) im_info = [im_tensor.shape[2], im_tensor.shape[3], im_scale] new_rec['boxes'] = roi_rec['boxes'].copy() * im_scale new_rec['im_info'] = im_info processed_roidb.append(new_rec) return processed_ims, processed_ref_ims, processed_eq_flags, processed_roidb def resize(im, target_size, max_size, stride=0, interpolation = cv2.INTER_LINEAR): """ only resize input image to target size and return scale :param im: BGR image input by opencv :param target_size: one dimensional size (the short side) :param max_size: one dimensional max size (the long side) :param stride: if given, pad the image to designated stride :param interpolation: if given, using given interpolation method to resize image :return: """ im_shape = im.shape im_size_min = np.min(im_shape[0:2]) im_size_max = np.max(im_shape[0:2]) im_scale = float(target_size) / float(im_size_min) # prevent bigger axis from being more than max_size: if np.round(im_scale * im_size_max) > max_size: im_scale = float(max_size) / float(im_size_max) im = cv2.resize(im, None, None, fx=im_scale, fy=im_scale, interpolation=interpolation) if stride == 0: return im, im_scale else: # pad to product of stride im_height = int(np.ceil(im.shape[0] / float(stride)) * stride) im_width = int(np.ceil(im.shape[1] / float(stride)) * stride) im_channel = im.shape[2] padded_im = np.zeros((im_height, im_width, im_channel)) padded_im[:im.shape[0], :im.shape[1], :] = im return padded_im, im_scale def transform(im, pixel_means): """ transform into mxnet tensor substract pixel size and transform to correct format :param im: [height, width, channel] in BGR :param pixel_means: [B, G, R pixel means] :return: [batch, channel, height, width] """ im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1])) for i in range(3): im_tensor[0, i, :, :] = im[:, :, 2 - i] - pixel_means[2 - i] return im_tensor def transform_seg_gt(gt): """ transform segmentation gt image into mxnet tensor :param gt: [height, width, channel = 1] :return: [batch, channel = 1, height, width] """ gt_tensor = np.zeros((1, 1, gt.shape[0], gt.shape[1])) gt_tensor[0, 0, :, :] = gt[:, :] return gt_tensor def transform_inverse(im_tensor, pixel_means): """ transform from mxnet im_tensor to ordinary RGB image im_tensor is limited to one image :param im_tensor: [batch, channel, height, width] :param pixel_means: [B, G, R pixel means] :return: im [height, width, channel(RGB)] """ assert im_tensor.shape[0] == 1 im_tensor = im_tensor.copy() # put channel back channel_swap = (0, 2, 3, 1) im_tensor = im_tensor.transpose(channel_swap) im = im_tensor[0] assert im.shape[2] == 3 im += pixel_means[[2, 1, 0]] im = im.astype(np.uint8) return im def tensor_vstack(tensor_list, pad=0): """ vertically stack tensors :param tensor_list: list of tensor to be stacked vertically :param pad: label to pad with :return: tensor with max shape """ ndim = len(tensor_list[0].shape) dtype = tensor_list[0].dtype islice = tensor_list[0].shape[0] dimensions = [] first_dim = sum([tensor.shape[0] for tensor in tensor_list]) dimensions.append(first_dim) for dim in range(1, ndim): dimensions.append(max([tensor.shape[dim] for tensor in tensor_list])) if pad == 0: all_tensor = np.zeros(tuple(dimensions), dtype=dtype) elif pad == 1: all_tensor = np.ones(tuple(dimensions), dtype=dtype) else: all_tensor = np.full(tuple(dimensions), pad, dtype=dtype) if ndim == 1: for ind, tensor in enumerate(tensor_list): all_tensor[ind*islice:(ind+1)*islice] = tensor elif ndim == 2: for ind, tensor in enumerate(tensor_list): all_tensor[ind*islice:(ind+1)*islice, :tensor.shape[1]] = tensor elif ndim == 3: for ind, tensor in enumerate(tensor_list): all_tensor[ind*islice:(ind+1)*islice, :tensor.shape[1], :tensor.shape[2]] = tensor elif ndim == 4: for ind, tensor in enumerate(tensor_list): all_tensor[ind*islice:(ind+1)*islice, :tensor.shape[1], :tensor.shape[2], :tensor.shape[3]] = tensor else: raise Exception('Sorry, unimplemented.') return all_tensor ================================================ FILE: lib/utils/image_processing.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong # -------------------------------------------------------- import numpy as np import cv2 def resize(im, target_size, max_size): """ only resize input image to target size and return scale :param im: BGR image input by opencv :param target_size: one dimensional size (the short side) :param max_size: one dimensional max size (the long side) :return: """ im_shape = im.shape im_size_min = np.min(im_shape[0:2]) im_size_max = np.max(im_shape[0:2]) im_scale = float(target_size) / float(im_size_min) # prevent bigger axis from being more than max_size: if np.round(im_scale * im_size_max) > max_size: im_scale = float(max_size) / float(im_size_max) im = cv2.resize(im, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR) return im, im_scale def transform(im, pixel_means, need_mean=False): """ transform into mxnet tensor subtract pixel size and transform to correct format :param im: [height, width, channel] in BGR :param pixel_means: [[[R, G, B pixel means]]] :return: [batch, channel, height, width] """ assert False, "shouldn't reach here." im = im.copy() im[:, :, (0, 1, 2)] = im[:, :, (2, 1, 0)] im = im.astype(float) if need_mean: im -= pixel_means im_tensor = im[np.newaxis, :] # put channel first channel_swap = (0, 3, 1, 2) im_tensor = im_tensor.transpose(channel_swap) return im_tensor def transform_inverse(im_tensor, pixel_means): """ transform from mxnet im_tensor to ordinary RGB image im_tensor is limited to one image :param im_tensor: [batch, channel, height, width] :param pixel_means: [[[R, G, B pixel means]]] :return: im [height, width, channel(RGB)] """ assert im_tensor.shape[0] == 1 im_tensor = im_tensor.copy() # put channel back channel_swap = (0, 2, 3, 1) im_tensor = im_tensor.transpose(channel_swap) im = im_tensor[0] assert im.shape[2] == 3 im += pixel_means im = im.astype(np.uint8) return im def tensor_vstack(tensor_list, pad=0): """ vertically stack tensors :param tensor_list: list of tensor to be stacked vertically :param pad: label to pad with :return: tensor with max shape """ ndim = len(tensor_list[0].shape) if ndim == 1: return np.hstack(tensor_list) dimensions = [0] for dim in range(1, ndim): dimensions.append(max([tensor.shape[dim] for tensor in tensor_list])) for ind, tensor in enumerate(tensor_list): pad_shape = [(0, 0)] for dim in range(1, ndim): pad_shape.append((0, dimensions[dim] - tensor.shape[dim])) tensor_list[ind] = np.lib.pad(tensor, pad_shape, 'constant', constant_values=pad) all_tensor = np.vstack(tensor_list) return all_tensor ================================================ FILE: lib/utils/load_data.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong # -------------------------------------------------------- import numpy as np from dataset import * def load_gt_roidb(dataset_name, image_set_name, root_path, dataset_path, result_path=None, flip=False): """ load ground truth roidb """ imdb = eval(dataset_name)(image_set_name, root_path, dataset_path, result_path) roidb = imdb.gt_roidb() if flip: roidb = imdb.append_flipped_images(roidb) return roidb def load_proposal_roidb(dataset_name, image_set_name, root_path, dataset_path, result_path=None, proposal='rpn', append_gt=True, flip=False): """ load proposal roidb (append_gt when training) """ imdb = eval(dataset_name)(image_set_name, root_path, dataset_path, result_path) gt_roidb = imdb.gt_roidb() roidb = eval('imdb.' + proposal + '_roidb')(gt_roidb, append_gt) if flip: roidb = imdb.append_flipped_images(roidb) return roidb def merge_roidb(roidbs): """ roidb are list, concat them together """ roidb = roidbs[0] for r in roidbs[1:]: roidb.extend(r) return roidb def filter_roidb(roidb, config): """ remove roidb entries without usable rois """ def is_valid(entry): """ valid images have at least 1 fg or bg roi """ overlaps = entry['max_overlaps'] fg_inds = np.where(overlaps >= config.TRAIN.FG_THRESH)[0] bg_inds = np.where((overlaps < config.TRAIN.BG_THRESH_HI) & (overlaps >= config.TRAIN.BG_THRESH_LO))[0] valid = len(fg_inds) > 0 or len(bg_inds) > 0 return valid num = len(roidb) filtered_roidb = [entry for entry in roidb if is_valid(entry)] num_after = len(filtered_roidb) print 'filtered %d roidb entries: %d -> %d' % (num - num_after, num, num_after) return filtered_roidb def load_gt_segdb(dataset_name, image_set_name, root_path, dataset_path, result_path=None, flip=False): """ load ground truth segdb """ imdb = eval(dataset_name)(image_set_name, root_path, dataset_path, result_path) segdb = imdb.gt_segdb() if flip: segdb = imdb.append_flipped_images_for_segmentation(segdb) return segdb def merge_segdb(segdbs): """ segdb are list, concat them together """ segdb = segdbs[0] for r in segdbs[1:]: segdb.extend(r) return segdb ================================================ FILE: lib/utils/load_model.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong # -------------------------------------------------------- import mxnet as mx def load_checkpoint(prefix, epoch): """ Load model checkpoint from file. :param prefix: Prefix of model name. :param epoch: Epoch number of model we would like to load. :return: (arg_params, aux_params) arg_params : dict of str to NDArray Model parameter, dict of name to NDArray of net's weights. aux_params : dict of str to NDArray Model parameter, dict of name to NDArray of net's auxiliary states. """ save_dict = mx.nd.load('%s-%04d.params' % (prefix, epoch)) arg_params = {} aux_params = {} for k, v in save_dict.items(): tp, name = k.split(':', 1) if tp == 'arg': arg_params[name] = v if tp == 'aux': aux_params[name] = v return arg_params, aux_params def convert_context(params, ctx): """ :param params: dict of str to NDArray :param ctx: the context to convert to :return: dict of str of NDArray with context ctx """ new_params = dict() for k, v in params.items(): new_params[k] = v.as_in_context(ctx) return new_params def load_param(prefix, epoch, convert=False, ctx=None, process=False): """ wrapper for load checkpoint :param prefix: Prefix of model name. :param epoch: Epoch number of model we would like to load. :param convert: reference model should be converted to GPU NDArray first :param ctx: if convert then ctx must be designated. :param process: model should drop any test :return: (arg_params, aux_params) """ arg_params, aux_params = load_checkpoint(prefix, epoch) if convert: if ctx is None: ctx = mx.cpu() arg_params = convert_context(arg_params, ctx) aux_params = convert_context(aux_params, ctx) if process: tests = [k for k in arg_params.keys() if '_test' in k] for test in tests: arg_params[test.replace('_test', '')] = arg_params.pop(test) return arg_params, aux_params ================================================ FILE: lib/utils/lr_scheduler.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong # -------------------------------------------------------- import logging from mxnet.lr_scheduler import LRScheduler class WarmupMultiFactorScheduler(LRScheduler): """Reduce learning rate in factor at steps specified in a list Assume the weight has been updated by n times, then the learning rate will be base_lr * factor^(sum((step/n)<=1)) # step is an array Parameters ---------- step: list of int schedule learning rate after n updates factor: float the factor for reducing the learning rate """ def __init__(self, step, factor=1, warmup=False, warmup_lr=0, warmup_step=0): super(WarmupMultiFactorScheduler, self).__init__() assert isinstance(step, list) and len(step) >= 1 for i, _step in enumerate(step): if i != 0 and step[i] <= step[i-1]: raise ValueError("Schedule step must be an increasing integer list") if _step < 1: raise ValueError("Schedule step must be greater or equal than 1 round") if factor > 1.0: raise ValueError("Factor must be no more than 1 to make lr reduce") self.step = step self.cur_step_ind = 0 self.factor = factor self.count = 0 self.warmup = warmup self.warmup_lr = warmup_lr self.warmup_step = warmup_step def __call__(self, num_update): """ Call to schedule current learning rate Parameters ---------- num_update: int the maximal number of updates applied to a weight. """ # NOTE: use while rather than if (for continuing training via load_epoch) if self.warmup and num_update < self.warmup_step: return self.warmup_lr while self.cur_step_ind <= len(self.step)-1: if num_update > self.step[self.cur_step_ind]: self.count = self.step[self.cur_step_ind] self.cur_step_ind += 1 self.base_lr *= self.factor logging.info("Update[%d]: Change learning rate to %0.5e", num_update, self.base_lr) else: return self.base_lr return self.base_lr ================================================ FILE: lib/utils/roidb.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong # -------------------------------------------------------- """ roidb basic format [image_index]['boxes', 'gt_classes', 'gt_overlaps', 'flipped'] extended ['image', 'max_classes', 'max_overlaps', 'bbox_targets'] """ import cv2 import numpy as np from bbox.bbox_regression import compute_bbox_regression_targets def prepare_roidb(imdb, roidb, cfg): """ add image path, max_classes, max_overlaps to roidb :param imdb: image database, provide path :param roidb: roidb :return: None """ print 'prepare roidb' for i in range(len(roidb)): # image_index roidb[i]['image'] = imdb.image_path_from_index(imdb.image_set_index[i]) if cfg.TRAIN.ASPECT_GROUPING: size = cv2.imread(roidb[i]['image']).shape roidb[i]['height'] = size[0] roidb[i]['width'] = size[1] gt_overlaps = roidb[i]['gt_overlaps'].toarray() max_overlaps = gt_overlaps.max(axis=1) max_classes = gt_overlaps.argmax(axis=1) roidb[i]['max_overlaps'] = max_overlaps roidb[i]['max_classes'] = max_classes # background roi => background class zero_indexes = np.where(max_overlaps == 0)[0] assert all(max_classes[zero_indexes] == 0) # foreground roi => foreground class nonzero_indexes = np.where(max_overlaps > 0)[0] assert all(max_classes[nonzero_indexes] != 0) ================================================ FILE: lib/utils/save_model.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong # -------------------------------------------------------- import mxnet as mx def save_checkpoint(prefix, epoch, arg_params, aux_params): """Checkpoint the model data into file. :param prefix: Prefix of model name. :param epoch: The epoch number of the model. :param arg_params: dict of str to NDArray Model parameter, dict of name to NDArray of net's weights. :param aux_params: dict of str to NDArray Model parameter, dict of name to NDArray of net's auxiliary states. :return: None prefix-epoch.params will be saved for parameters. """ save_dict = {('arg:%s' % k) : v for k, v in arg_params.items()} save_dict.update({('aux:%s' % k) : v for k, v in aux_params.items()}) param_name = '%s-%04d.params' % (prefix, epoch) mx.nd.save(param_name, save_dict) ================================================ FILE: lib/utils/show_boxes.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yi Li, Haocheng Zhang, Xizhou Zhu # -------------------------------------------------------- import matplotlib.pyplot as plt import cv2 import random def show_boxes(im, dets, classes, scale = 1.0): plt.cla() plt.axis("off") plt.imshow(im) for cls_idx, cls_name in enumerate(classes): cls_dets = dets[cls_idx] for det in cls_dets: bbox = det[:4] * scale color = (random.random(), random.random(), random.random()) rect = plt.Rectangle((bbox[0], bbox[1]), bbox[2] - bbox[0], bbox[3] - bbox[1], fill=False, edgecolor=color, linewidth=2.5) plt.gca().add_patch(rect) if cls_dets.shape[1] == 5: score = det[-1] plt.gca().text(bbox[0], bbox[1], '{:s} {:.3f}'.format(cls_name, score), bbox=dict(facecolor=color, alpha=0.5), fontsize=9, color='white') plt.show() return im def draw_boxes(im, dets, classes, scale = 1.0): color_white = (255, 255, 255) for cls_idx, cls_name in enumerate(classes): cls_dets = dets[cls_idx] for det in cls_dets: bbox = det[:4] * scale bbox = map(int, bbox) color = (random.randint(0, 256), random.randint(0, 256), random.randint(0, 256)) cv2.rectangle(im, (bbox[0], bbox[1]), (bbox[2], bbox[3]), color=color, thickness=3) if cls_dets.shape[1] == 5: score = det[-1] cv2.putText(im, '%s %.3f' % (cls_name, score), (bbox[0], bbox[1]+10), color=color_white, fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=1, thickness=2) return im ================================================ FILE: lib/utils/symbol.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong # -------------------------------------------------------- import numpy as np class Symbol: def __init__(self): self.arg_shape_dict = None self.out_shape_dict = None self.aux_shape_dict = None self.sym = None @property def symbol(self): return self.sym def get_symbol(self, cfg, is_train=True): """ return a generated symbol, it also need to be assigned to self.sym """ raise NotImplementedError() def init_weights(self, cfg, arg_params, aux_params): raise NotImplementedError() def get_msra_std(self, shape): fan_in = float(shape[1]) if len(shape) > 2: fan_in *= np.prod(shape[2:]) print(np.sqrt(2 / fan_in)) return np.sqrt(2 / fan_in) def infer_shape(self, data_shape_dict): # infer shape arg_shape, out_shape, aux_shape = self.sym.infer_shape(**data_shape_dict) self.arg_shape_dict = dict(zip(self.sym.list_arguments(), arg_shape)) self.out_shape_dict = dict(zip(self.sym.list_outputs(), out_shape)) self.aux_shape_dict = dict(zip(self.sym.list_auxiliary_states(), aux_shape)) def check_parameter_shapes(self, arg_params, aux_params, data_shape_dict, is_train=True): for k in self.sym.list_arguments(): if k in data_shape_dict or (False if is_train else 'label' in k): continue assert k in arg_params, k + ' not initialized' assert arg_params[k].shape == self.arg_shape_dict[k], \ 'shape inconsistent for ' + k + ' inferred ' + str(self.arg_shape_dict[k]) + ' provided ' + str( arg_params[k].shape) for k in self.sym.list_auxiliary_states(): assert k in aux_params, k + ' not initialized' assert aux_params[k].shape == self.aux_shape_dict[k], \ 'shape inconsistent for ' + k + ' inferred ' + str(self.aux_shape_dict[k]) + ' provided ' + str( aux_params[k].shape) ================================================ FILE: lib/utils/tictoc.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong # -------------------------------------------------------- import time def tic(): import time global startTime_for_tictoc startTime_for_tictoc = time.time() return startTime_for_tictoc def toc(): if 'startTime_for_tictoc' in globals(): endTime = time.time() return endTime - startTime_for_tictoc else: return None ================================================ FILE: rfcn/__init__.py ================================================ ================================================ FILE: rfcn/_init_paths.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Xizhou Zhu # -------------------------------------------------------- import os.path as osp import sys def add_path(path): if path not in sys.path: sys.path.insert(0, path) this_dir = osp.dirname(__file__) lib_path = osp.join(this_dir, '..', 'lib') add_path(lib_path) ================================================ FILE: rfcn/config/__init__.py ================================================ ================================================ FILE: rfcn/config/config.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Xizhou Zhu, Yuwen Xiong, Bin Xiao # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- import yaml import numpy as np from easydict import EasyDict as edict config = edict() config.MXNET_VERSION = '' config.output_path = '' config.symbol = '' config.gpus = '' config.CLASS_AGNOSTIC = True config.SCALES = [(600, 1000)] # first is scale (the shorter side); second is max size # default training config.default = edict() config.default.frequent = 20 config.default.kvstore = 'device' # network related params config.network = edict() config.network.pretrained = '' config.network.pretrained_epoch = 0 config.network.PIXEL_MEANS = np.array([0, 0, 0]) config.network.IMAGE_STRIDE = 0 config.network.RPN_FEAT_STRIDE = 16 config.network.RCNN_FEAT_STRIDE = 16 config.network.FIXED_PARAMS = ['gamma', 'beta'] config.network.ANCHOR_SCALES = (8, 16, 32) config.network.ANCHOR_RATIOS = (0.5, 1, 2) config.network.NORMALIZE_RPN = True config.network.ANCHOR_MEANS = (0.0, 0.0, 0.0, 0.0) config.network.ANCHOR_STDS = (0.1, 0.1, 0.4, 0.4) config.network.NUM_ANCHORS = len(config.network.ANCHOR_SCALES) * len(config.network.ANCHOR_RATIOS) # dataset related params config.dataset = edict() config.dataset.dataset = 'ImageNetVID' config.dataset.image_set = 'DET_train_30classes+VID_train_15frames' config.dataset.test_image_set = 'VID_val_frames' config.dataset.root_path = './data' config.dataset.dataset_path = './data/ILSVRC2015' config.dataset.NUM_CLASSES = 31 config.TRAIN = edict() config.TRAIN.lr = 0 config.TRAIN.lr_step = '' config.TRAIN.lr_factor = 0.1 config.TRAIN.warmup = False config.TRAIN.warmup_lr = 0 config.TRAIN.warmup_step = 0 config.TRAIN.momentum = 0.9 config.TRAIN.wd = 0.0005 config.TRAIN.begin_epoch = 0 config.TRAIN.end_epoch = 0 config.TRAIN.model_prefix = '' # whether resume training config.TRAIN.RESUME = False # whether flip image config.TRAIN.FLIP = True # whether shuffle image config.TRAIN.SHUFFLE = True # whether use OHEM config.TRAIN.ENABLE_OHEM = False # size of images for each device, 2 for rcnn, 1 for rpn and e2e config.TRAIN.BATCH_IMAGES = 2 # e2e changes behavior of anchor loader and metric config.TRAIN.END2END = False # group images with similar aspect ratio config.TRAIN.ASPECT_GROUPING = True # R-CNN # rcnn rois batch size config.TRAIN.BATCH_ROIS = 128 config.TRAIN.BATCH_ROIS_OHEM = 128 # rcnn rois sampling params config.TRAIN.FG_FRACTION = 0.25 config.TRAIN.FG_THRESH = 0.5 config.TRAIN.BG_THRESH_HI = 0.5 config.TRAIN.BG_THRESH_LO = 0.0 # rcnn bounding box regression params config.TRAIN.BBOX_REGRESSION_THRESH = 0.5 config.TRAIN.BBOX_WEIGHTS = np.array([1.0, 1.0, 1.0, 1.0]) # RPN anchor loader # rpn anchors batch size config.TRAIN.RPN_BATCH_SIZE = 256 # rpn anchors sampling params config.TRAIN.RPN_FG_FRACTION = 0.5 config.TRAIN.RPN_POSITIVE_OVERLAP = 0.7 config.TRAIN.RPN_NEGATIVE_OVERLAP = 0.3 config.TRAIN.RPN_CLOBBER_POSITIVES = False # rpn bounding box regression params config.TRAIN.RPN_BBOX_WEIGHTS = (1.0, 1.0, 1.0, 1.0) config.TRAIN.RPN_POSITIVE_WEIGHT = -1.0 # used for end2end training # RPN proposal config.TRAIN.CXX_PROPOSAL = True config.TRAIN.RPN_NMS_THRESH = 0.7 config.TRAIN.RPN_PRE_NMS_TOP_N = 12000 config.TRAIN.RPN_POST_NMS_TOP_N = 2000 config.TRAIN.RPN_MIN_SIZE = config.network.RPN_FEAT_STRIDE # approximate bounding box regression config.TRAIN.BBOX_NORMALIZATION_PRECOMPUTED = True config.TRAIN.BBOX_MEANS = (0.0, 0.0, 0.0, 0.0) config.TRAIN.BBOX_STDS = (0.1, 0.1, 0.2, 0.2) config.TEST = edict() # R-CNN testing # use rpn to generate proposal config.TEST.HAS_RPN = False # size of images for each device config.TEST.BATCH_IMAGES = 1 # RPN proposal config.TEST.CXX_PROPOSAL = True config.TEST.RPN_NMS_THRESH = 0.7 config.TEST.RPN_PRE_NMS_TOP_N = 6000 config.TEST.RPN_POST_NMS_TOP_N = 300 config.TEST.RPN_MIN_SIZE = config.network.RPN_FEAT_STRIDE # RCNN nms config.TEST.NMS = 0.3 config.TEST.max_per_image = 300 # Test Model Epoch config.TEST.test_epoch = 0 def update_config(config_file): exp_config = None with open(config_file) as f: exp_config = edict(yaml.load(f)) for k, v in exp_config.items(): if k in config: if isinstance(v, dict): if k == 'TRAIN': if 'BBOX_WEIGHTS' in v: v['BBOX_WEIGHTS'] = np.array(v['BBOX_WEIGHTS']) elif k == 'network': if 'PIXEL_MEANS' in v: v['PIXEL_MEANS'] = np.array(v['PIXEL_MEANS']) for vk, vv in v.items(): config[k][vk] = vv else: if k == 'SCALES': config[k][0] = (tuple(v)) else: config[k] = v else: raise ValueError("key must exist in config.py") ================================================ FILE: rfcn/core/DataParallelExecutorGroup.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- import logging import numpy as np from mxnet import context as ctx from mxnet import ndarray as nd from mxnet.io import DataDesc from mxnet.executor_manager import _split_input_slice def _load_general(data, targets, major_axis): """Load a list of arrays into a list of arrays specified by slices""" for d_src, d_targets in zip(data, targets): if isinstance(d_targets, nd.NDArray): d_src.copyto(d_targets) elif isinstance(d_src, (list, tuple)): for src, dst in zip(d_src, d_targets): src.copyto(dst) else: raise NotImplementedError def _load_data(batch, targets, major_axis): """Load data into sliced arrays""" _load_general(batch.data, targets, major_axis) def _load_label(batch, targets, major_axis): """Load label into sliced arrays""" _load_general(batch.label, targets, major_axis) def _merge_multi_context(outputs, major_axis): """Merge outputs that lives on multiple context into one, so that they look like living on one context. """ rets = [] for tensors, axis in zip(outputs, major_axis): if axis >= 0: rets.append(nd.concatenate(tensors, axis=axis, always_copy=False)) else: # negative axis means the there is no batch_size axis, and all the # results should be the same on each device. We simply take the # first one, without checking they are actually the same rets.append(tensors[0]) return rets class DataParallelExecutorGroup(object): """DataParallelExecutorGroup is a group of executors that lives on a group of devices. This is a helper class used to implement data parallelization. Each mini-batch will be split and run on the devices. Parameters ---------- symbol : Symbol The common symbolic computation graph for all executors. contexts : list A list of contexts. workload : list If not `None`, could be a list of numbers that specify the workload to be assigned to different context. Larger number indicate heavier workload. data_shapes : list Should be a list of (name, shape) tuples, for the shapes of data. Note the order is important and should be the same as the order that the `DataIter` provide the data. label_shapes : list Should be a list of (name, shape) tuples, for the shapes of label. Note the order is important and should be the same as the order that the `DataIter` provide the label. param_names : list A list of strings, indicating the names of parameters (e.g. weights, filters, etc.) in the computation graph. for_training : bool Indicate whether the executors should be bind for training. When not doing training, the memory for gradients will not be allocated. inputs_need_grad : bool Indicate whether the gradients for the input data should be computed. This is currently not used. It will be useful for implementing composition of modules. shared_group : DataParallelExecutorGroup Default is `None`. This is used in bucketing. When not `None`, it should be a executor group corresponding to a different bucket. In other words, it will correspond to a different symbol but with the same set of parameters (e.g. unrolled RNNs with different lengths). In this case, many memory will be shared. logger : Logger Default is `logging`. fixed_param_names: list of str Indicate parameters to be fixed during training. Parameters in this list will not allocate space for gradient, nor do gradient calculation. grad_req : str, list of str, dict of str to str Requirement for gradient accumulation. Can be 'write', 'add', or 'null' (default to 'write'). Can be specified globally (str) or for each argument (list, dict). """ def __init__(self, symbol, contexts, workload, data_shapes, label_shapes, param_names, for_training, inputs_need_grad, shared_group=None, logger=logging, fixed_param_names=None, grad_req='write', state_names=None): self.param_names = param_names self.arg_names = symbol.list_arguments() self.aux_names = symbol.list_auxiliary_states() self.symbol = symbol self.contexts = contexts self.workload = workload self.for_training = for_training self.inputs_need_grad = inputs_need_grad self.logger = logger #In the future we should have a better way to profile memory per device (haibin) # self._total_exec_bytes = 0 self.fixed_param_names = fixed_param_names if self.fixed_param_names is None: self.fixed_param_names = [] self.state_names = state_names if self.state_names is None: self.state_names = [] if not for_training: grad_req = 'null' # data_shapes = [x if isinstance(x, DataDesc) else DataDesc(*x) for x in data_shapes] # if label_shapes is not None: # label_shapes = [x if isinstance(x, DataDesc) else DataDesc(*x) for x in label_shapes] data_names = [x.name for x in data_shapes[0]] if isinstance(grad_req, str): self.grad_req = {} for k in self.arg_names: if k in self.param_names: self.grad_req[k] = 'null' if k in self.fixed_param_names else grad_req elif k in data_names: self.grad_req[k] = grad_req if self.inputs_need_grad else 'null' else: self.grad_req[k] = 'null' elif isinstance(grad_req, (list, tuple)): assert len(grad_req) == len(self.arg_names) self.grad_req = dict(zip(self.arg_names, grad_req)) elif isinstance(grad_req, dict): self.grad_req = {} for k in self.arg_names: if k in self.param_names: self.grad_req[k] = 'null' if k in self.fixed_param_names else 'write' elif k in data_names: self.grad_req[k] = 'write' if self.inputs_need_grad else 'null' else: self.grad_req[k] = 'null' self.grad_req.update(grad_req) else: raise ValueError("grad_req must be one of str, list, tuple, or dict.") if shared_group is not None: self.shared_data_arrays = shared_group.shared_data_arrays else: self.shared_data_arrays = [{} for _ in contexts] # initialize some instance variables self.batch_size = len(data_shapes) self.slices = None self.execs = [] self._default_execs = None self.data_arrays = None self.label_arrays = None self.param_arrays = None self.state_arrays = None self.grad_arrays = None self.aux_arrays = None self.input_grad_arrays = None self.data_shapes = None self.label_shapes = None self.data_layouts = None self.label_layouts = None self.output_layouts = [DataDesc.get_batch_axis(self.symbol[name].attr('__layout__')) for name in self.symbol.list_outputs()] self.bind_exec(data_shapes, label_shapes, shared_group) def decide_slices(self, data_shapes): """Decide the slices for each context according to the workload. Parameters ---------- data_shapes : list list of (name, shape) specifying the shapes for the input data or label. """ assert len(data_shapes) > 0 major_axis = [DataDesc.get_batch_axis(x.layout) for x in data_shapes] for (name, shape), axis in zip(data_shapes, major_axis): if axis == -1: continue batch_size = shape[axis] if self.batch_size is not None: assert batch_size == self.batch_size, ("all data must have the same batch size: " + ("batch_size = %d, but " % self.batch_size) + ("%s has shape %s" % (name, shape))) else: self.batch_size = batch_size self.slices = _split_input_slice(self.batch_size, self.workload) return major_axis def _collect_arrays(self): """Collect internal arrays from executors.""" # convenient data structures self.data_arrays = [[e.arg_dict[name] for name, _ in self.data_shapes[0]] for e in self.execs] self.state_arrays = [[e.arg_dict[name] for e in self.execs] for name in self.state_names] if self.label_shapes is not None: self.label_arrays = [[e.arg_dict[name] for name, _ in self.label_shapes[0]] for e in self.execs] else: self.label_arrays = None self.param_arrays = [[exec_.arg_arrays[i] for exec_ in self.execs] for i, name in enumerate(self.arg_names) if name in self.param_names] if self.for_training: self.grad_arrays = [[exec_.grad_arrays[i] for exec_ in self.execs] for i, name in enumerate(self.arg_names) if name in self.param_names] else: self.grad_arrays = None data_names = [x[0] for x in self.data_shapes] if self.inputs_need_grad: self.input_grad_arrays = [[exec_.grad_arrays[i] for exec_ in self.execs] for i, name in enumerate(self.arg_names) if name in data_names] else: self.input_grad_arrays = None self.aux_arrays = [[exec_.aux_arrays[i] for exec_ in self.execs] for i in range(len(self.aux_names))] def bind_exec(self, data_shapes, label_shapes, shared_group=None, reshape=False): """Bind executors on their respective devices. Parameters ---------- data_shapes : list label_shapes : list shared_group : DataParallelExecutorGroup reshape : bool """ assert reshape or not self.execs for i in range(len(self.contexts)): data_shapes_i = data_shapes[i] if label_shapes is not None: label_shapes_i = label_shapes[i] else: label_shapes_i = [] if reshape: self.execs[i] = self._default_execs[i].reshape( allow_up_sizing=True, **dict(data_shapes_i + label_shapes_i)) else: self.execs.append(self._bind_ith_exec(i, data_shapes_i, label_shapes_i, shared_group)) self.data_shapes = data_shapes self.label_shapes = label_shapes self._collect_arrays() def reshape(self, data_shapes, label_shapes): """Reshape executors. Parameters ---------- data_shapes : list label_shapes : list """ if self._default_execs is None: self._default_execs = [i for i in self.execs] for i in range(len(self.contexts)): self.execs[i] = self._default_execs[i].reshape( allow_up_sizing=True, **dict(data_shapes[i] + (label_shapes[i] if label_shapes is not None else [])) ) self.data_shapes = data_shapes self.label_shapes = label_shapes self._collect_arrays() def set_params(self, arg_params, aux_params): """Assign, i.e. copy parameters to all the executors. Parameters ---------- arg_params : dict A dictionary of name to `NDArray` parameter mapping. aux_params : dict A dictionary of name to `NDArray` auxiliary variable mapping. """ for exec_ in self.execs: exec_.copy_params_from(arg_params, aux_params) def get_params(self, arg_params, aux_params): """ Copy data from each executor to `arg_params` and `aux_params`. Parameters ---------- arg_params : list of NDArray target parameter arrays aux_params : list of NDArray target aux arrays Notes ----- - This function will inplace update the NDArrays in arg_params and aux_params. """ for name, block in zip(self.param_names, self.param_arrays): weight = sum(w.copyto(ctx.cpu()) for w in block) / len(block) weight.astype(arg_params[name].dtype).copyto(arg_params[name]) for name, block in zip(self.aux_names, self.aux_arrays): weight = sum(w.copyto(ctx.cpu()) for w in block) / len(block) weight.astype(aux_params[name].dtype).copyto(aux_params[name]) def forward(self, data_batch, is_train=None): """Split `data_batch` according to workload and run forward on each devices. Parameters ---------- data_batch : DataBatch Or could be any object implementing similar interface. is_train : bool The hint for the backend, indicating whether we are during training phase. Default is `None`, then the value `self.for_training` will be used. Returns ------- """ _load_data(data_batch, self.data_arrays, self.data_layouts) if is_train is None: is_train = self.for_training if self.label_arrays is not None: assert not is_train or data_batch.label if data_batch.label: _load_label(data_batch, self.label_arrays, self.label_layouts) for exec_ in self.execs: exec_.forward(is_train=is_train) def get_outputs(self, merge_multi_context=True): """Get outputs of the previous forward computation. Parameters ---------- merge_multi_context : bool Default is `True`. In the case when data-parallelism is used, the outputs will be collected from multiple devices. A `True` value indicate that we should merge the collected results so that they look like from a single executor. Returns ------- If `merge_multi_context` is `True`, it is like `[out1, out2]`. Otherwise, it is like `[[out1_dev1, out1_dev2], [out2_dev1, out2_dev2]]`. All the output elements are `NDArray`. """ outputs = [[exec_.outputs[i] for exec_ in self.execs] for i in range(len(self.execs[0].outputs))] if merge_multi_context: outputs = _merge_multi_context(outputs, self.output_layouts) return outputs def get_states(self, merge_multi_context=True): """Get states from all devices Parameters ---------- merge_multi_context : bool Default is `True`. In the case when data-parallelism is used, the states will be collected from multiple devices. A `True` value indicate that we should merge the collected results so that they look like from a single executor. Returns ------- If `merge_multi_context` is `True`, it is like `[out1, out2]`. Otherwise, it is like `[[out1_dev1, out1_dev2], [out2_dev1, out2_dev2]]`. All the output elements are `NDArray`. """ assert not merge_multi_context, \ "merge_multi_context=True is not supported for get_states yet." return self.state_arrays def set_states(self, states=None, value=None): """Set value for states. Only one of states & value can be specified. Parameters ---------- states : list of list of NDArrays source states arrays formatted like [[state1_dev1, state1_dev2], [state2_dev1, state2_dev2]]. value : number a single scalar value for all state arrays. """ if states is not None: assert value is None, "Only one of states & value can be specified." _load_general(states, self.state_arrays, (0,)*len(states)) else: assert value is not None, "At least one of states & value must be specified." assert states is None, "Only one of states & value can be specified." for d_dst in self.state_arrays: for dst in d_dst: dst[:] = value def get_input_grads(self, merge_multi_context=True): """Get the gradients with respect to the inputs of the module. Parameters ---------- merge_multi_context : bool Default is `True`. In the case when data-parallelism is used, the outputs will be collected from multiple devices. A `True` value indicate that we should merge the collected results so that they look like from a single executor. Returns ------- If `merge_multi_context` is `True`, it is like `[grad1, grad2]`. Otherwise, it is like `[[grad1_dev1, grad1_dev2], [grad2_dev1, grad2_dev2]]`. All the output elements are `NDArray`. """ assert self.inputs_need_grad if merge_multi_context: return _merge_multi_context(self.input_grad_arrays, self.data_layouts) return self.input_grad_arrays def backward(self, out_grads=None): """Run backward on all devices. A backward should be called after a call to the forward function. Backward cannot be called unless `self.for_training` is `True`. Parameters ---------- out_grads : NDArray or list of NDArray, optional Gradient on the outputs to be propagated back. This parameter is only needed when bind is called on outputs that are not a loss function. """ assert self.for_training, 're-bind with for_training=True to run backward' if out_grads is None: out_grads = [] for i, exec_ in enumerate(self.execs): out_grads_slice = [] exec_.backward(out_grads=out_grads_slice) def update_metric(self, eval_metric, labels): """Accumulate the performance according to `eval_metric` on all devices. Parameters ---------- eval_metric : EvalMetric The metric used for evaluation. labels : list of NDArray Typically comes from `label` of a `DataBatch`. """ for texec, labels in zip(self.execs, labels): eval_metric.update(labels, texec.outputs) def _bind_ith_exec(self, i, data_shapes, label_shapes, shared_group): """Internal utility function to bind the i-th executor. """ shared_exec = None if shared_group is None else shared_group.execs[i] context = self.contexts[i] shared_data_arrays = self.shared_data_arrays[i] input_shapes = dict(data_shapes) if label_shapes is not None: input_shapes.update(dict(label_shapes)) arg_shapes, _, aux_shapes = self.symbol.infer_shape(**input_shapes) assert arg_shapes is not None, "shape inference failed" input_types = {x.name: x.dtype for x in data_shapes} if label_shapes is not None: input_types.update({x.name: x.dtype for x in label_shapes}) arg_types, _, aux_types = self.symbol.infer_type(**input_types) assert arg_types is not None, "type inference failed" arg_arrays = [] grad_arrays = {} if self.for_training else None def _get_or_reshape(name, shared_data_arrays, arg_shape, arg_type, context, logger): """Internal helper to get a memory block or re-use by re-shaping""" if name in shared_data_arrays: arg_arr = shared_data_arrays[name] if np.prod(arg_arr.shape) >= np.prod(arg_shape): # nice, we can directly re-use this data blob assert arg_arr.dtype == arg_type arg_arr = arg_arr.reshape(arg_shape) else: logger.warning(('bucketing: data "%s" has a shape %s' % (name, arg_shape)) + (', which is larger than already allocated ') + ('shape %s' % (arg_arr.shape,)) + ('. Need to re-allocate. Consider putting ') + ('default_bucket_key to') + (' be the bucket taking the largest input for better ') + ('memory sharing.')) arg_arr = nd.zeros(arg_shape, context, dtype=arg_type) # replace existing shared array because the new one is bigger shared_data_arrays[name] = arg_arr else: arg_arr = nd.zeros(arg_shape, context, dtype=arg_type) shared_data_arrays[name] = arg_arr return arg_arr # create or borrow arguments and gradients for j in range(len(self.arg_names)): name = self.arg_names[j] if name in self.param_names: # model parameters if shared_exec is None: arg_arr = nd.zeros(arg_shapes[j], context, dtype=arg_types[j]) if self.grad_req[name] != 'null': grad_arr = nd.zeros(arg_shapes[j], context, dtype=arg_types[j]) grad_arrays[name] = grad_arr else: arg_arr = shared_exec.arg_dict[name] assert arg_arr.shape == arg_shapes[j] assert arg_arr.dtype == arg_types[j] if self.grad_req[name] != 'null': grad_arrays[name] = shared_exec.grad_dict[name] else: # data, label, or states arg_arr = _get_or_reshape(name, shared_data_arrays, arg_shapes[j], arg_types[j], context, self.logger) # data might also need grad if inputs_need_grad is True if self.grad_req[name] != 'null': grad_arrays[name] = _get_or_reshape('grad of ' + name, shared_data_arrays, arg_shapes[j], arg_types[j], context, self.logger) arg_arrays.append(arg_arr) # create or borrow aux variables if shared_exec is None: aux_arrays = [nd.zeros(s, context, dtype=t) for s, t in zip(aux_shapes, aux_types)] else: for j, arr in enumerate(shared_exec.aux_arrays): assert aux_shapes[j] == arr.shape assert aux_types[j] == arr.dtype aux_arrays = shared_exec.aux_arrays[:] executor = self.symbol.bind(ctx=context, args=arg_arrays, args_grad=grad_arrays, aux_states=aux_arrays, grad_req=self.grad_req, shared_exec=shared_exec) # Get the total bytes allocated for this executor return executor def _sliced_shape(self, shapes, i, major_axis): """Get the sliced shapes for the i-th executor. Parameters ---------- shapes : list of (str, tuple) The original (name, shape) pairs. i : int Which executor we are dealing with. """ sliced_shapes = [] for desc, axis in zip(shapes, major_axis): shape = list(desc.shape) if axis >= 0: shape[axis] = self.slices[i].stop - self.slices[i].start sliced_shapes.append(DataDesc(desc.name, tuple(shape), desc.dtype, desc.layout)) return sliced_shapes def install_monitor(self, mon): """Install monitor on all executors""" for exe in self.execs: mon.install(exe) ================================================ FILE: rfcn/core/__init__.py ================================================ ================================================ FILE: rfcn/core/callback.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- import time import logging import mxnet as mx class Speedometer(object): def __init__(self, batch_size, frequent=50): self.batch_size = batch_size self.frequent = frequent self.init = False self.tic = 0 self.last_count = 0 def __call__(self, param): """Callback to Show speed.""" count = param.nbatch if self.last_count > count: self.init = False self.last_count = count if self.init: if count % self.frequent == 0: speed = self.frequent * self.batch_size / (time.time() - self.tic) s = '' if param.eval_metric is not None: name, value = param.eval_metric.get() s = "Epoch[%d] Batch [%d]\tSpeed: %.2f samples/sec\tTrain-" % (param.epoch, count, speed) for n, v in zip(name, value): s += "%s=%f,\t" % (n, v) else: s = "Iter[%d] Batch [%d]\tSpeed: %.2f samples/sec" % (param.epoch, count, speed) logging.info(s) print(s) self.tic = time.time() else: self.init = True self.tic = time.time() def do_checkpoint(prefix, means, stds): def _callback(iter_no, sym, arg, aux): weight = arg['rfcn_bbox_weight'] bias = arg['rfcn_bbox_bias'] repeat = bias.shape[0] / means.shape[0] arg['rfcn_bbox_weight_test'] = weight * mx.nd.repeat(mx.nd.array(stds), repeats=repeat).reshape((bias.shape[0], 1, 1, 1)) arg['rfcn_bbox_bias_test'] = arg['rfcn_bbox_bias'] * mx.nd.repeat(mx.nd.array(stds), repeats=repeat) + mx.nd.repeat(mx.nd.array(means), repeats=repeat) mx.model.save_checkpoint(prefix, iter_no + 1, sym, arg, aux) arg.pop('rfcn_bbox_weight_test') arg.pop('rfcn_bbox_bias_test') return _callback ================================================ FILE: rfcn/core/loader.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- import numpy as np import mxnet as mx from mxnet.executor_manager import _split_input_slice from config.config import config from utils.image import tensor_vstack from rpn.rpn import get_rpn_testbatch, get_rpn_batch, assign_anchor from rcnn import get_rcnn_testbatch, get_rcnn_batch class TestLoader(mx.io.DataIter): def __init__(self, roidb, config, batch_size=1, shuffle=False, has_rpn=False): super(TestLoader, self).__init__() # save parameters as properties self.cfg = config self.roidb = roidb self.batch_size = batch_size self.shuffle = shuffle self.has_rpn = has_rpn # infer properties from roidb self.size = len(self.roidb) self.index = np.arange(self.size) # decide data and label names (only for training) if has_rpn: self.data_name = ['data', 'im_info'] else: self.data_name = ['data', 'rois'] self.label_name = None # status variable for synchronization between get_data and get_label self.cur = 0 self.data = None self.label = [] self.im_info = None # get first batch to fill in provide_data and provide_label self.reset() self.get_batch() @property def provide_data(self): return [[(k, v.shape) for k, v in zip(self.data_name, idata)] for idata in self.data] @property def provide_label(self): return [None for _ in range(len(self.data))] @property def provide_data_single(self): return [(k, v.shape) for k, v in zip(self.data_name, self.data[0])] @property def provide_label_single(self): return None def reset(self): self.cur = 0 if self.shuffle: np.random.shuffle(self.index) def iter_next(self): return self.cur < self.size def next(self): if self.iter_next(): self.get_batch() self.cur += self.batch_size return self.im_info, mx.io.DataBatch(data=self.data, label=self.label, pad=self.getpad(), index=self.getindex(), provide_data=self.provide_data, provide_label=self.provide_label) else: raise StopIteration def getindex(self): return self.cur / self.batch_size def getpad(self): if self.cur + self.batch_size > self.size: return self.cur + self.batch_size - self.size else: return 0 def get_batch(self): cur_from = self.cur cur_to = min(cur_from + self.batch_size, self.size) roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)] if self.has_rpn: data, label, im_info = get_rpn_testbatch(roidb, self.cfg) else: data, label, im_info = get_rcnn_testbatch(roidb, self.cfg) self.data = [[mx.nd.array(idata[name]) for name in self.data_name] for idata in data] self.im_info = im_info class AnchorLoader(mx.io.DataIter): def __init__(self, feat_sym, roidb, cfg, batch_size=1, shuffle=False, ctx=None, work_load_list=None, feat_stride=16, anchor_scales=(8, 16, 32), anchor_ratios=(0.5, 1, 2), allowed_border=0, aspect_grouping=False, normalize_target=False, bbox_mean=(0.0, 0.0, 0.0, 0.0), bbox_std=(0.1, 0.1, 0.4, 0.4)): """ This Iter will provide roi data to Fast R-CNN network :param feat_sym: to infer shape of assign_output :param roidb: must be preprocessed :param batch_size: must divide BATCH_SIZE(128) :param shuffle: bool :param ctx: list of contexts :param work_load_list: list of work load :param aspect_grouping: group images with similar aspects :param normalize_target: normalize rpn target :param bbox_mean: anchor target mean :param bbox_std: anchor target std :return: AnchorLoader """ super(AnchorLoader, self).__init__() # save parameters as properties self.feat_sym = feat_sym self.roidb = roidb self.cfg = cfg self.batch_size = batch_size self.shuffle = shuffle self.ctx = ctx if self.ctx is None: self.ctx = [mx.cpu()] self.work_load_list = work_load_list self.feat_stride = feat_stride self.anchor_scales = anchor_scales self.anchor_ratios = anchor_ratios self.allowed_border = allowed_border self.aspect_grouping = aspect_grouping self.normalize_target = normalize_target self.bbox_mean = bbox_mean self.bbox_std = bbox_std # infer properties from roidb self.size = len(roidb) self.index = np.arange(self.size) # decide data and label names if config.TRAIN.END2END: self.data_name = ['data', 'im_info', 'gt_boxes'] else: self.data_name = ['data'] self.label_name = ['label', 'bbox_target', 'bbox_weight'] # status variable for synchronization between get_data and get_label self.cur = 0 self.batch = None self.data = None self.label = None # get first batch to fill in provide_data and provide_label self.reset() self.get_batch_individual() @property def provide_data(self): return [[(k, v.shape) for k, v in zip(self.data_name, self.data[i])] for i in xrange(len(self.data))] @property def provide_label(self): return [[(k, v.shape) for k, v in zip(self.label_name, self.label[i])] for i in xrange(len(self.data))] @property def provide_data_single(self): return [(k, v.shape) for k, v in zip(self.data_name, self.data[0])] @property def provide_label_single(self): return [(k, v.shape) for k, v in zip(self.label_name, self.label[0])] def reset(self): self.cur = 0 if self.shuffle: if self.aspect_grouping: widths = np.array([r['width'] for r in self.roidb]) heights = np.array([r['height'] for r in self.roidb]) horz = (widths >= heights) vert = np.logical_not(horz) horz_inds = np.where(horz)[0] vert_inds = np.where(vert)[0] inds = np.hstack((np.random.permutation(horz_inds), np.random.permutation(vert_inds))) extra = inds.shape[0] % self.batch_size inds_ = np.reshape(inds[:-extra], (-1, self.batch_size)) row_perm = np.random.permutation(np.arange(inds_.shape[0])) inds[:-extra] = np.reshape(inds_[row_perm, :], (-1,)) self.index = inds else: np.random.shuffle(self.index) def iter_next(self): return self.cur + self.batch_size <= self.size def next(self): if self.iter_next(): self.get_batch_individual() self.cur += self.batch_size return mx.io.DataBatch(data=self.data, label=self.label, pad=self.getpad(), index=self.getindex(), provide_data=self.provide_data, provide_label=self.provide_label) else: raise StopIteration def getindex(self): return self.cur / self.batch_size def getpad(self): if self.cur + self.batch_size > self.size: return self.cur + self.batch_size - self.size else: return 0 def infer_shape(self, max_data_shape=None, max_label_shape=None): """ Return maximum data and label shape for single gpu """ if max_data_shape is None: max_data_shape = [] if max_label_shape is None: max_label_shape = [] max_shapes = dict(max_data_shape + max_label_shape) input_batch_size = max_shapes['data'][0] im_info = [[max_shapes['data'][2], max_shapes['data'][3], 1.0]] _, feat_shape, _ = self.feat_sym.infer_shape(**max_shapes) label = assign_anchor(feat_shape[0], np.zeros((0, 5)), im_info, self.cfg, self.feat_stride, self.anchor_scales, self.anchor_ratios, self.allowed_border, self.normalize_target, self.bbox_mean, self.bbox_std) label = [label[k] for k in self.label_name] label_shape = [(k, tuple([input_batch_size] + list(v.shape[1:]))) for k, v in zip(self.label_name, label)] return max_data_shape, label_shape def get_batch(self): # slice roidb cur_from = self.cur cur_to = min(cur_from + self.batch_size, self.size) roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)] # decide multi device slice work_load_list = self.work_load_list ctx = self.ctx if work_load_list is None: work_load_list = [1] * len(ctx) assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \ "Invalid settings for work load. " slices = _split_input_slice(self.batch_size, work_load_list) # get testing data for multigpu data_list = [] label_list = [] for islice in slices: iroidb = [roidb[i] for i in range(islice.start, islice.stop)] data, label = get_rpn_batch(iroidb, self.cfg) data_list.append(data) label_list.append(label) # pad data first and then assign anchor (read label) data_tensor = tensor_vstack([batch['data'] for batch in data_list]) for data, data_pad in zip(data_list, data_tensor): data['data'] = data_pad[np.newaxis, :] new_label_list = [] for data, label in zip(data_list, label_list): # infer label shape data_shape = {k: v.shape for k, v in data.items()} del data_shape['im_info'] _, feat_shape, _ = self.feat_sym.infer_shape(**data_shape) feat_shape = [int(i) for i in feat_shape[0]] # add gt_boxes to data for e2e data['gt_boxes'] = label['gt_boxes'][np.newaxis, :, :] # assign anchor for label label = assign_anchor(feat_shape, label['gt_boxes'], data['im_info'], self.cfg, self.feat_stride, self.anchor_scales, self.anchor_ratios, self.allowed_border, self.normalize_target, self.bbox_mean, self.bbox_std) new_label_list.append(label) all_data = dict() for key in self.data_name: all_data[key] = tensor_vstack([batch[key] for batch in data_list]) all_label = dict() for key in self.label_name: pad = -1 if key == 'label' else 0 all_label[key] = tensor_vstack([batch[key] for batch in new_label_list], pad=pad) self.data = [mx.nd.array(all_data[key]) for key in self.data_name] self.label = [mx.nd.array(all_label[key]) for key in self.label_name] def get_batch_individual(self): cur_from = self.cur cur_to = min(cur_from + self.batch_size, self.size) roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)] # decide multi device slice work_load_list = self.work_load_list ctx = self.ctx if work_load_list is None: work_load_list = [1] * len(ctx) assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \ "Invalid settings for work load. " slices = _split_input_slice(self.batch_size, work_load_list) rst = [] for idx, islice in enumerate(slices): iroidb = [roidb[i] for i in range(islice.start, islice.stop)] rst.append(self.parfetch(iroidb)) all_data = [_['data'] for _ in rst] all_label = [_['label'] for _ in rst] self.data = [[mx.nd.array(data[key]) for key in self.data_name] for data in all_data] self.label = [[mx.nd.array(label[key]) for key in self.label_name] for label in all_label] def parfetch(self, iroidb): # get testing data for multigpu data, label = get_rpn_batch(iroidb, self.cfg) data_shape = {k: v.shape for k, v in data.items()} del data_shape['im_info'] _, feat_shape, _ = self.feat_sym.infer_shape(**data_shape) feat_shape = [int(i) for i in feat_shape[0]] # add gt_boxes to data for e2e data['gt_boxes'] = label['gt_boxes'][np.newaxis, :, :] # assign anchor for label label = assign_anchor(feat_shape, label['gt_boxes'], data['im_info'], self.cfg, self.feat_stride, self.anchor_scales, self.anchor_ratios, self.allowed_border, self.normalize_target, self.bbox_mean, self.bbox_std) return {'data': data, 'label': label} ================================================ FILE: rfcn/core/metric.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- import mxnet as mx import numpy as np def get_rpn_names(): pred = ['rpn_cls_prob', 'rpn_bbox_loss'] label = ['rpn_label', 'rpn_bbox_target', 'rpn_bbox_weight'] return pred, label def get_rcnn_names(cfg): pred = ['rcnn_cls_prob', 'rcnn_bbox_loss'] label = ['rcnn_label', 'rcnn_bbox_target', 'rcnn_bbox_weight'] if cfg.TRAIN.ENABLE_OHEM or cfg.TRAIN.END2END: pred.append('rcnn_label') if cfg.TRAIN.END2END: rpn_pred, rpn_label = get_rpn_names() pred = rpn_pred + pred label = rpn_label return pred, label class RPNAccMetric(mx.metric.EvalMetric): def __init__(self): super(RPNAccMetric, self).__init__('RPNAcc') self.pred, self.label = get_rpn_names() def update(self, labels, preds): pred = preds[self.pred.index('rpn_cls_prob')] label = labels[self.label.index('rpn_label')] # pred (b, c, p) or (b, c, h, w) pred_label = mx.ndarray.argmax_channel(pred).asnumpy().astype('int32') pred_label = pred_label.reshape((pred_label.shape[0], -1)) # label (b, p) label = label.asnumpy().astype('int32') # filter with keep_inds keep_inds = np.where(label != -1) pred_label = pred_label[keep_inds] label = label[keep_inds] self.sum_metric += np.sum(pred_label.flat == label.flat) self.num_inst += len(pred_label.flat) class RCNNAccMetric(mx.metric.EvalMetric): def __init__(self, cfg): super(RCNNAccMetric, self).__init__('RCNNAcc') self.e2e = cfg.TRAIN.END2END self.ohem = cfg.TRAIN.ENABLE_OHEM self.pred, self.label = get_rcnn_names(cfg) def update(self, labels, preds): pred = preds[self.pred.index('rcnn_cls_prob')] if self.ohem or self.e2e: label = preds[self.pred.index('rcnn_label')] else: label = labels[self.label.index('rcnn_label')] last_dim = pred.shape[-1] pred_label = pred.asnumpy().reshape(-1, last_dim).argmax(axis=1).astype('int32') label = label.asnumpy().reshape(-1,).astype('int32') # filter with keep_inds keep_inds = np.where(label != -1) pred_label = pred_label[keep_inds] label = label[keep_inds] self.sum_metric += np.sum(pred_label.flat == label.flat) self.num_inst += len(pred_label.flat) class RPNLogLossMetric(mx.metric.EvalMetric): def __init__(self): super(RPNLogLossMetric, self).__init__('RPNLogLoss') self.pred, self.label = get_rpn_names() def update(self, labels, preds): pred = preds[self.pred.index('rpn_cls_prob')] label = labels[self.label.index('rpn_label')] # label (b, p) label = label.asnumpy().astype('int32').reshape((-1)) # pred (b, c, p) or (b, c, h, w) --> (b, p, c) --> (b*p, c) pred = pred.asnumpy().reshape((pred.shape[0], pred.shape[1], -1)).transpose((0, 2, 1)) pred = pred.reshape((label.shape[0], -1)) # filter with keep_inds keep_inds = np.where(label != -1)[0] label = label[keep_inds] cls = pred[keep_inds, label] cls += 1e-14 cls_loss = -1 * np.log(cls) cls_loss = np.sum(cls_loss) self.sum_metric += cls_loss self.num_inst += label.shape[0] class RCNNLogLossMetric(mx.metric.EvalMetric): def __init__(self, cfg): super(RCNNLogLossMetric, self).__init__('RCNNLogLoss') self.e2e = cfg.TRAIN.END2END self.ohem = cfg.TRAIN.ENABLE_OHEM self.pred, self.label = get_rcnn_names(cfg) def update(self, labels, preds): pred = preds[self.pred.index('rcnn_cls_prob')] if self.ohem or self.e2e: label = preds[self.pred.index('rcnn_label')] else: label = labels[self.label.index('rcnn_label')] last_dim = pred.shape[-1] pred = pred.asnumpy().reshape(-1, last_dim) label = label.asnumpy().reshape(-1,).astype('int32') # filter with keep_inds keep_inds = np.where(label != -1)[0] label = label[keep_inds] cls = pred[keep_inds, label] cls += 1e-14 cls_loss = -1 * np.log(cls) cls_loss = np.sum(cls_loss) self.sum_metric += cls_loss self.num_inst += label.shape[0] class RPNL1LossMetric(mx.metric.EvalMetric): def __init__(self): super(RPNL1LossMetric, self).__init__('RPNL1Loss') self.pred, self.label = get_rpn_names() def update(self, labels, preds): bbox_loss = preds[self.pred.index('rpn_bbox_loss')].asnumpy() # calculate num_inst (average on those kept anchors) label = labels[self.label.index('rpn_label')].asnumpy() num_inst = np.sum(label != -1) self.sum_metric += np.sum(bbox_loss) self.num_inst += num_inst class RCNNL1LossMetric(mx.metric.EvalMetric): def __init__(self, cfg): super(RCNNL1LossMetric, self).__init__('RCNNL1Loss') self.e2e = cfg.TRAIN.END2END self.ohem = cfg.TRAIN.ENABLE_OHEM self.pred, self.label = get_rcnn_names(cfg) def update(self, labels, preds): bbox_loss = preds[self.pred.index('rcnn_bbox_loss')].asnumpy() if self.ohem: label = preds[self.pred.index('rcnn_label')].asnumpy() else: if self.e2e: label = preds[self.pred.index('rcnn_label')].asnumpy() else: label = labels[self.label.index('rcnn_label')].asnumpy() # calculate num_inst (average on those kept anchors) num_inst = np.sum(label != -1) self.sum_metric += np.sum(bbox_loss) self.num_inst += num_inst ================================================ FILE: rfcn/core/module.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- """A `MutableModule` implement the `BaseModule` API, and allows input shape varying with training iterations. If shapes vary, executors will rebind, using shared arrays from the initial module binded with maximum shape. """ import time import logging import warnings from mxnet import context as ctx from mxnet.initializer import Uniform, InitDesc from mxnet.module.base_module import BaseModule, _check_input_names, _parse_data_desc, _as_list from mxnet.model import _create_kvstore, _initialize_kvstore, _update_params, _update_params_on_kvstore, load_checkpoint, BatchEndParam from mxnet import metric from .DataParallelExecutorGroup import DataParallelExecutorGroup from mxnet import ndarray as nd from mxnet import optimizer as opt class Module(BaseModule): """Module is a basic module that wrap a `Symbol`. It is functionally the same as the `FeedForward` model, except under the module API. Parameters ---------- symbol : Symbol data_names : list of str Default is `('data')` for a typical model used in image classification. label_names : list of str Default is `('softmax_label')` for a typical model used in image classification. logger : Logger Default is `logging`. context : Context or list of Context Default is `cpu()`. work_load_list : list of number Default `None`, indicating uniform workload. fixed_param_names: list of str Default `None`, indicating no network parameters are fixed. state_names : list of str states are similar to data and label, but not provided by data iterator. Instead they are initialized to 0 and can be set by set_states() """ def __init__(self, symbol, data_names=('data',), label_names=('softmax_label',), logger=logging, context=ctx.cpu(), work_load_list=None, fixed_param_names=None, state_names=None): super(Module, self).__init__(logger=logger) if isinstance(context, ctx.Context): context = [context] self._context = context if work_load_list is None: work_load_list = [1] * len(self._context) assert len(work_load_list) == len(self._context) self._work_load_list = work_load_list self._symbol = symbol data_names = list(data_names) if data_names is not None else [] label_names = list(label_names) if label_names is not None else [] state_names = list(state_names) if state_names is not None else [] fixed_param_names = list(fixed_param_names) if fixed_param_names is not None else [] _check_input_names(symbol, data_names, "data", True) _check_input_names(symbol, label_names, "label", False) _check_input_names(symbol, state_names, "state", True) _check_input_names(symbol, fixed_param_names, "fixed_param", True) arg_names = symbol.list_arguments() input_names = data_names + label_names + state_names self._param_names = [x for x in arg_names if x not in input_names] self._fixed_param_names = fixed_param_names self._aux_names = symbol.list_auxiliary_states() self._data_names = data_names self._label_names = label_names self._state_names = state_names self._output_names = symbol.list_outputs() self._arg_params = None self._aux_params = None self._params_dirty = False self._optimizer = None self._kvstore = None self._update_on_kvstore = None self._updater = None self._preload_opt_states = None self._grad_req = None self._exec_group = None self._data_shapes = None self._label_shapes = None @staticmethod def load(prefix, epoch, load_optimizer_states=False, **kwargs): """Create a model from previously saved checkpoint. Parameters ---------- prefix : str path prefix of saved model files. You should have "prefix-symbol.json", "prefix-xxxx.params", and optionally "prefix-xxxx.states", where xxxx is the epoch number. epoch : int epoch to load. load_optimizer_states : bool whether to load optimizer states. Checkpoint needs to have been made with save_optimizer_states=True. data_names : list of str Default is `('data')` for a typical model used in image classification. label_names : list of str Default is `('softmax_label')` for a typical model used in image classification. logger : Logger Default is `logging`. context : Context or list of Context Default is `cpu()`. work_load_list : list of number Default `None`, indicating uniform workload. fixed_param_names: list of str Default `None`, indicating no network parameters are fixed. """ sym, args, auxs = load_checkpoint(prefix, epoch) mod = Module(symbol=sym, **kwargs) mod._arg_params = args mod._aux_params = auxs mod.params_initialized = True if load_optimizer_states: mod._preload_opt_states = '%s-%04d.states'%(prefix, epoch) return mod def save_checkpoint(self, prefix, epoch, save_optimizer_states=False): """Save current progress to checkpoint. Use mx.callback.module_checkpoint as epoch_end_callback to save during training. Parameters ---------- prefix : str The file prefix to checkpoint to epoch : int The current epoch number save_optimizer_states : bool Whether to save optimizer states for continue training """ self._symbol.save('%s-symbol.json'%prefix) param_name = '%s-%04d.params' % (prefix, epoch) self.save_params(param_name) logging.info('Saved checkpoint to \"%s\"', param_name) if save_optimizer_states: state_name = '%s-%04d.states' % (prefix, epoch) self.save_optimizer_states(state_name) logging.info('Saved optimizer state to \"%s\"', state_name) def _reset_bind(self): """Internal function to reset binded state.""" self.binded = False self._exec_group = None self._data_shapes = None self._label_shapes = None @property def data_names(self): """A list of names for data required by this module.""" return self._data_names @property def label_names(self): """A list of names for labels required by this module.""" return self._label_names @property def output_names(self): """A list of names for the outputs of this module.""" return self._output_names @property def data_shapes(self): """Get data shapes. Returns ------- A list of `(name, shape)` pairs. """ assert self.binded return self._data_shapes @property def label_shapes(self): """Get label shapes. Returns ------- A list of `(name, shape)` pairs. The return value could be `None` if the module does not need labels, or if the module is not binded for training (in this case, label information is not available). """ assert self.binded return self._label_shapes @property def output_shapes(self): """Get output shapes. Returns ------- A list of `(name, shape)` pairs. """ assert self.binded return self._exec_group.get_output_shapes() def get_params(self): """Get current parameters. Returns ------- `(arg_params, aux_params)`, each a dictionary of name to parameters (in `NDArray`) mapping. """ assert self.binded and self.params_initialized if self._params_dirty: self._sync_params_from_devices() return (self._arg_params, self._aux_params) def init_params(self, initializer=Uniform(0.01), arg_params=None, aux_params=None, allow_missing=False, force_init=False): """Initialize the parameters and auxiliary states. Parameters ---------- initializer : Initializer Called to initialize parameters if needed. arg_params : dict If not None, should be a dictionary of existing arg_params. Initialization will be copied from that. aux_params : dict If not None, should be a dictionary of existing aux_params. Initialization will be copied from that. allow_missing : bool If true, params could contain missing values, and the initializer will be called to fill those missing params. force_init : bool If true, will force re-initialize even if already initialized. """ if self.params_initialized and not force_init: warnings.warn("Parameters already initialized and force_init=False. " "init_params call ignored.", stacklevel=2) return assert self.binded, 'call bind before initializing the parameters' def _impl(name, arr, cache): """Internal helper for parameter initialization""" if cache is not None: if name in cache: cache_arr = cache[name] # just in case the cached array is just the target itself if cache_arr is not arr: cache_arr.copyto(arr) else: if not allow_missing: raise RuntimeError("%s is not presented" % name) if initializer != None: initializer(name, arr) else: initializer(name, arr) attrs = self._symbol.attr_dict() for name, arr in self._arg_params.items(): desc = InitDesc(name, attrs.get(name, None)) _impl(desc, arr, arg_params) for name, arr in self._aux_params.items(): desc = InitDesc(name, attrs.get(name, None)) _impl(desc, arr, aux_params) self.params_initialized = True self._params_dirty = False # copy the initialized parameters to devices self._exec_group.set_params(self._arg_params, self._aux_params) def set_params(self, arg_params, aux_params, allow_missing=False, force_init=True): """Assign parameter and aux state values. Parameters ---------- arg_params : dict Dictionary of name to value (`NDArray`) mapping. aux_params : dict Dictionary of name to value (`NDArray`) mapping. allow_missing : bool If true, params could contain missing values, and the initializer will be called to fill those missing params. force_init : bool If true, will force re-initialize even if already initialized. Examples -------- An example of setting module parameters:: >>> sym, arg_params, aux_params = \ >>> mx.model.load_checkpoint(model_prefix, n_epoch_load) >>> mod.set_params(arg_params=arg_params, aux_params=aux_params) """ if not allow_missing: self.init_params(initializer=None, arg_params=arg_params, aux_params=aux_params, allow_missing=allow_missing, force_init=force_init) return if self.params_initialized and not force_init: warnings.warn("Parameters already initialized and force_init=False. " "set_params call ignored.", stacklevel=2) return self._exec_group.set_params(arg_params, aux_params) # because we didn't update self._arg_params, they are dirty now. self._params_dirty = True self.params_initialized = True def bind(self, data_shapes, label_shapes=None, for_training=True, inputs_need_grad=False, force_rebind=False, shared_module=None, grad_req='write'): """Bind the symbols to construct executors. This is necessary before one can perform computation with the module. Parameters ---------- data_shapes : list of (str, tuple) Typically is `data_iter.provide_data`. label_shapes : list of (str, tuple) Typically is `data_iter.provide_label`. for_training : bool Default is `True`. Whether the executors should be bind for training. inputs_need_grad : bool Default is `False`. Whether the gradients to the input data need to be computed. Typically this is not needed. But this might be needed when implementing composition of modules. force_rebind : bool Default is `False`. This function does nothing if the executors are already binded. But with this `True`, the executors will be forced to rebind. shared_module : Module Default is `None`. This is used in bucketing. When not `None`, the shared module essentially corresponds to a different bucket -- a module with different symbol but with the same sets of parameters (e.g. unrolled RNNs with different lengths). """ # force rebinding is typically used when one want to switch from # training to prediction phase. if force_rebind: self._reset_bind() if self.binded: self.logger.warning('Already binded, ignoring bind()') return self.for_training = for_training self.inputs_need_grad = inputs_need_grad self.binded = True self._grad_req = grad_req if not for_training: assert not inputs_need_grad else: pass # this is not True, as some module might not contains a loss function # that consumes the labels # assert label_shapes is not None # self._data_shapes, self._label_shapes = _parse_data_desc( # self.data_names, self.label_names, data_shapes, label_shapes) self._data_shapes, self._label_shapes = zip(*[_parse_data_desc(self.data_names, self.label_names, data_shape, label_shape) for data_shape, label_shape in zip(data_shapes, label_shapes)]) if self._label_shapes.count(None) == len(self._label_shapes): self._label_shapes = None if shared_module is not None: assert isinstance(shared_module, Module) and \ shared_module.binded and shared_module.params_initialized shared_group = shared_module._exec_group else: shared_group = None self._exec_group = DataParallelExecutorGroup(self._symbol, self._context, self._work_load_list, self._data_shapes, self._label_shapes, self._param_names, for_training, inputs_need_grad, shared_group, logger=self.logger, fixed_param_names=self._fixed_param_names, grad_req=grad_req, state_names=self._state_names) # self._total_exec_bytes = self._exec_group._total_exec_bytes if shared_module is not None: self.params_initialized = True self._arg_params = shared_module._arg_params self._aux_params = shared_module._aux_params elif self.params_initialized: # if the parameters are already initialized, we are re-binding # so automatically copy the already initialized params self._exec_group.set_params(self._arg_params, self._aux_params) else: assert self._arg_params is None and self._aux_params is None param_arrays = [ nd.zeros(x[0].shape, dtype=x[0].dtype) for x in self._exec_group.param_arrays ] self._arg_params = {name:arr for name, arr in zip(self._param_names, param_arrays)} aux_arrays = [ nd.zeros(x[0].shape, dtype=x[0].dtype) for x in self._exec_group.aux_arrays ] self._aux_params = {name:arr for name, arr in zip(self._aux_names, aux_arrays)} if shared_module is not None and shared_module.optimizer_initialized: self.borrow_optimizer(shared_module) def reshape(self, data_shapes, label_shapes=None): """Reshape the module for new input shapes. Parameters ---------- data_shapes : list of (str, tuple) Typically is `data_iter.provide_data`. label_shapes : list of (str, tuple) Typically is `data_iter.provide_label`. """ assert self.binded # self._data_shapes, self._label_shapes = _parse_data_desc( # self.data_names, self.label_names, data_shapes, label_shapes) self._data_shapes, self._label_shapes = zip(*[_parse_data_desc(self.data_names, self.label_names, data_shape, label_shape) for data_shape, label_shape in zip(data_shapes, label_shapes)]) self._exec_group.reshape(self._data_shapes, self._label_shapes) def init_optimizer(self, kvstore='local', optimizer='sgd', optimizer_params=(('learning_rate', 0.01),), force_init=False): """Install and initialize optimizers. Parameters ---------- kvstore : str or KVStore Default `'local'`. optimizer : str or Optimizer Default `'sgd'` optimizer_params : dict Default `(('learning_rate', 0.01),)`. The default value is not a dictionary, just to avoid pylint warning of dangerous default values. force_init : bool Default `False`, indicating whether we should force re-initializing the optimizer in the case an optimizer is already installed. """ assert self.binded and self.params_initialized if self.optimizer_initialized and not force_init: self.logger.warning('optimizer already initialized, ignoring...') return (kvstore, update_on_kvstore) = \ _create_kvstore(kvstore, len(self._context), self._arg_params) batch_size = self._exec_group.batch_size if kvstore and 'dist' in kvstore.type and '_sync' in kvstore.type: batch_size *= kvstore.num_workers rescale_grad = 1.0/batch_size if isinstance(optimizer, str): idx2name = {} if update_on_kvstore: idx2name.update(enumerate(self._exec_group.param_names)) else: for k in range(len(self._context)): idx2name.update({i*len(self._context)+k: n for i, n in enumerate(self._exec_group.param_names)}) optimizer_params = dict(optimizer_params) if 'rescale_grad' not in optimizer_params: optimizer_params['rescale_grad'] = rescale_grad optimizer = opt.create(optimizer, sym=self.symbol, param_idx2name=idx2name, **optimizer_params) else: assert isinstance(optimizer, opt.Optimizer) if optimizer.rescale_grad != rescale_grad: #pylint: disable=no-member warnings.warn( "Optimizer created manually outside Module but rescale_grad " + "is not normalized to 1.0/batch_size/num_workers (%s vs. %s). "%( optimizer.rescale_grad, rescale_grad) + "Is this intended?", stacklevel=2) self._optimizer = optimizer self._kvstore = kvstore self._update_on_kvstore = update_on_kvstore self._updater = None if kvstore: # copy initialized local parameters to kvstore _initialize_kvstore(kvstore=kvstore, param_arrays=self._exec_group.param_arrays, arg_params=self._arg_params, param_names=self._param_names, update_on_kvstore=update_on_kvstore) if update_on_kvstore: kvstore.set_optimizer(self._optimizer) else: self._updater = opt.get_updater(optimizer) self.optimizer_initialized = True if self._preload_opt_states is not None: self.load_optimizer_states(self._preload_opt_states) self._preload_opt_states = None def borrow_optimizer(self, shared_module): """Borrow optimizer from a shared module. Used in bucketing, where exactly the same optimizer (esp. kvstore) is used. Parameters ---------- shared_module : Module """ assert shared_module.optimizer_initialized self._optimizer = shared_module._optimizer self._kvstore = shared_module._kvstore self._update_on_kvstore = shared_module._update_on_kvstore self._updater = shared_module._updater self.optimizer_initialized = True def forward(self, data_batch, is_train=None): """Forward computation. Parameters ---------- data_batch : DataBatch Could be anything with similar API implemented. is_train : bool Default is `None`, which means `is_train` takes the value of `self.for_training`. """ assert self.binded and self.params_initialized self._exec_group.forward(data_batch, is_train) def backward(self, out_grads=None): """Backward computation. Parameters ---------- out_grads : NDArray or list of NDArray, optional Gradient on the outputs to be propagated back. This parameter is only needed when bind is called on outputs that are not a loss function. """ assert self.binded and self.params_initialized self._exec_group.backward(out_grads=out_grads) def update(self): """Update parameters according to the installed optimizer and the gradients computed in the previous forward-backward batch. """ assert self.binded and self.params_initialized and self.optimizer_initialized self._params_dirty = True if self._update_on_kvstore: _update_params_on_kvstore(self._exec_group.param_arrays, self._exec_group.grad_arrays, self._kvstore) else: _update_params(self._exec_group.param_arrays, self._exec_group.grad_arrays, updater=self._updater, num_device=len(self._context), kvstore=self._kvstore) def get_outputs(self, merge_multi_context=True): """Get outputs of the previous forward computation. Parameters ---------- merge_multi_context : bool Default is `True`. In the case when data-parallelism is used, the outputs will be collected from multiple devices. A `True` value indicate that we should merge the collected results so that they look like from a single executor. Returns ------- If `merge_multi_context` is `True`, it is like `[out1, out2]`. Otherwise, it is like `[[out1_dev1, out1_dev2], [out2_dev1, out2_dev2]]`. All the output elements are `NDArray`. """ assert self.binded and self.params_initialized return self._exec_group.get_outputs(merge_multi_context=merge_multi_context) def get_input_grads(self, merge_multi_context=True): """Get the gradients with respect to the inputs of the module. Parameters ---------- merge_multi_context : bool Default is `True`. In the case when data-parallelism is used, the outputs will be collected from multiple devices. A `True` value indicate that we should merge the collected results so that they look like from a single executor. Returns ------- If `merge_multi_context` is `True`, it is like `[grad1, grad2]`. Otherwise, it is like `[[grad1_dev1, grad1_dev2], [grad2_dev1, grad2_dev2]]`. All the output elements are `NDArray`. """ assert self.binded and self.params_initialized and self.inputs_need_grad return self._exec_group.get_input_grads(merge_multi_context=merge_multi_context) def get_states(self, merge_multi_context=True): """Get states from all devices Parameters ---------- merge_multi_context : bool Default is `True`. In the case when data-parallelism is used, the states will be collected from multiple devices. A `True` value indicate that we should merge the collected results so that they look like from a single executor. Returns ------- If `merge_multi_context` is `True`, it is like `[out1, out2]`. Otherwise, it is like `[[out1_dev1, out1_dev2], [out2_dev1, out2_dev2]]`. All the output elements are `NDArray`. """ assert self.binded and self.params_initialized return self._exec_group.get_states(merge_multi_context=merge_multi_context) def set_states(self, states=None, value=None): """Set value for states. Only one of states & value can be specified. Parameters ---------- states : list of list of NDArrays source states arrays formatted like [[state1_dev1, state1_dev2], [state2_dev1, state2_dev2]]. value : number a single scalar value for all state arrays. """ assert self.binded and self.params_initialized self._exec_group.set_states(states, value) def update_metric(self, eval_metric, labels): """Evaluate and accumulate evaluation metric on outputs of the last forward computation. Parameters ---------- eval_metric : EvalMetric labels : list of NDArray Typically `data_batch.label`. """ self._exec_group.update_metric(eval_metric, labels) def _sync_params_from_devices(self): """Synchronize parameters from devices to CPU. This function should be called after calling `update` that updates the parameters on the devices, before one can read the latest parameters from `self._arg_params` and `self._aux_params`. """ self._exec_group.get_params(self._arg_params, self._aux_params) self._params_dirty = False def save_optimizer_states(self, fname): """Save optimizer (updater) state to file Parameters ---------- fname : str Path to output states file. """ assert self.optimizer_initialized if self._update_on_kvstore: self._kvstore.save_optimizer_states(fname) else: with open(fname, 'wb') as fout: fout.write(self._updater.get_states()) def load_optimizer_states(self, fname): """Load optimizer (updater) state from file Parameters ---------- fname : str Path to input states file. """ assert self.optimizer_initialized if self._update_on_kvstore: self._kvstore.load_optimizer_states(fname) else: self._updater.set_states(open(fname, 'rb').read()) def install_monitor(self, mon): """ Install monitor on all executors """ assert self.binded self._exec_group.install_monitor(mon) class MutableModule(BaseModule): """A mutable module is a module that supports variable input data. Parameters ---------- symbol : Symbol data_names : list of str label_names : list of str logger : Logger context : Context or list of Context work_load_list : list of number max_data_shapes : list of (name, shape) tuple, designating inputs whose shape vary max_label_shapes : list of (name, shape) tuple, designating inputs whose shape vary fixed_param_prefix : list of str, indicating fixed parameters """ def __init__(self, symbol, data_names, label_names, logger=logging, context=ctx.cpu(), work_load_list=None, max_data_shapes=None, max_label_shapes=None, fixed_param_prefix=None): super(MutableModule, self).__init__(logger=logger) self._symbol = symbol self._data_names = data_names self._label_names = label_names self._context = context self._work_load_list = work_load_list self._curr_module = None self._max_data_shapes = max_data_shapes self._max_label_shapes = max_label_shapes self._fixed_param_prefix = fixed_param_prefix fixed_param_names = list() if fixed_param_prefix is not None: for name in self._symbol.list_arguments(): for prefix in self._fixed_param_prefix: if name.startswith(prefix): fixed_param_names.append(name) self._fixed_param_names = fixed_param_names self._preload_opt_states = None def _reset_bind(self): self.binded = False self._curr_module = None @property def data_names(self): return self._data_names @property def output_names(self): return self._symbol.list_outputs() @property def data_shapes(self): assert self.binded return self._curr_module.data_shapes @property def label_shapes(self): assert self.binded return self._curr_module.label_shapes @property def output_shapes(self): assert self.binded return self._curr_module.output_shapes def get_params(self): assert self.binded and self.params_initialized return self._curr_module.get_params() def init_params(self, initializer=Uniform(0.01), arg_params=None, aux_params=None, allow_missing=False, force_init=False): if self.params_initialized and not force_init: return assert self.binded, 'call bind before initializing the parameters' self._curr_module.init_params(initializer=initializer, arg_params=arg_params, aux_params=aux_params, allow_missing=allow_missing, force_init=force_init) self.params_initialized = True def bind(self, data_shapes, label_shapes=None, for_training=True, inputs_need_grad=False, force_rebind=False, shared_module=None, grad_req='write'): # in case we already initialized params, keep it if self.params_initialized: arg_params, aux_params = self.get_params() # force rebinding is typically used when one want to switch from # training to prediction phase. if force_rebind: self._reset_bind() if self.binded: self.logger.warning('Already binded, ignoring bind()') return assert shared_module is None, 'shared_module for MutableModule is not supported' self.for_training = for_training self.inputs_need_grad = inputs_need_grad self.binded = True max_shapes_dict = dict() if self._max_data_shapes is not None: max_shapes_dict.update(dict(self._max_data_shapes[0])) if self._max_label_shapes is not None: max_shapes_dict.update(dict(self._max_label_shapes[0])) max_data_shapes = list() for name, shape in data_shapes[0]: if name in max_shapes_dict: max_data_shapes.append((name, max_shapes_dict[name])) else: max_data_shapes.append((name, shape)) max_label_shapes = list() if not label_shapes.count(None) == len(label_shapes): for name, shape in label_shapes[0]: if name in max_shapes_dict: max_label_shapes.append((name, max_shapes_dict[name])) else: max_label_shapes.append((name, shape)) if len(max_label_shapes) == 0: max_label_shapes = None module = Module(self._symbol, self._data_names, self._label_names, logger=self.logger, context=self._context, work_load_list=self._work_load_list, fixed_param_names=self._fixed_param_names) module.bind([max_data_shapes for _ in xrange(len(self._context))], [max_label_shapes for _ in xrange(len(self._context))], for_training, inputs_need_grad, force_rebind=False, shared_module=None) self._curr_module = module # copy back saved params, if already initialized if self.params_initialized: self.set_params(arg_params, aux_params) def save_checkpoint(self, prefix, epoch, save_optimizer_states=False): """Save current progress to checkpoint. Use mx.callback.module_checkpoint as epoch_end_callback to save during training. Parameters ---------- prefix : str The file prefix to checkpoint to epoch : int The current epoch number save_optimizer_states : bool Whether to save optimizer states for continue training """ self._curr_module.save_checkpoint(prefix, epoch, save_optimizer_states) def init_optimizer(self, kvstore='local', optimizer='sgd', optimizer_params=(('learning_rate', 0.01),), force_init=False): assert self.binded and self.params_initialized if self.optimizer_initialized and not force_init: self.logger.warning('optimizer already initialized, ignoring.') return self._curr_module._preload_opt_states = self._preload_opt_states self._curr_module.init_optimizer(kvstore, optimizer, optimizer_params, force_init=force_init) self.optimizer_initialized = True def fit(self, train_data, eval_data=None, eval_metric='acc', epoch_end_callback=None, batch_end_callback=None, kvstore='local', optimizer='sgd', optimizer_params=(('learning_rate', 0.01),), eval_end_callback=None, eval_batch_end_callback=None, initializer=Uniform(0.01), arg_params=None, aux_params=None, allow_missing=False, force_rebind=False, force_init=False, begin_epoch=0, num_epoch=None, validation_metric=None, monitor=None, prefix=None): """Train the module parameters. Parameters ---------- train_data : DataIter eval_data : DataIter If not `None`, will be used as validation set and evaluate the performance after each epoch. eval_metric : str or EvalMetric Default `'acc'`. The performance measure used to display during training. epoch_end_callback : function or list of function Each callback will be called with the current `epoch`, `symbol`, `arg_params` and `aux_params`. batch_end_callback : function or list of function Each callback will be called with a `BatchEndParam`. kvstore : str or KVStore Default `'local'`. optimizer : str or Optimizer Default `'sgd'` optimizer_params : dict Default `(('learning_rate', 0.01),)`. The parameters for the optimizer constructor. The default value is not a `dict`, just to avoid pylint warning on dangerous default values. eval_end_callback : function or list of function These will be called at the end of each full evaluation, with the metrics over the entire evaluation set. eval_batch_end_callback : function or list of function These will be called at the end of each minibatch during evaluation initializer : Initializer Will be called to initialize the module parameters if not already initialized. arg_params : dict Default `None`, if not `None`, should be existing parameters from a trained model or loaded from a checkpoint (previously saved model). In this case, the value here will be used to initialize the module parameters, unless they are already initialized by the user via a call to `init_params` or `fit`. `arg_params` has higher priority to `initializer`. aux_params : dict Default `None`. Similar to `arg_params`, except for auxiliary states. allow_missing : bool Default `False`. Indicate whether we allow missing parameters when `arg_params` and `aux_params` are not `None`. If this is `True`, then the missing parameters will be initialized via the `initializer`. force_rebind : bool Default `False`. Whether to force rebinding the executors if already binded. force_init : bool Default `False`. Indicate whether we should force initialization even if the parameters are already initialized. begin_epoch : int Default `0`. Indicate the starting epoch. Usually, if we are resuming from a checkpoint saved at a previous training phase at epoch N, then we should specify this value as N+1. num_epoch : int Number of epochs to run training. Examples -------- An example of using fit for training:: >>> #Assume training dataIter and validation dataIter are ready >>> mod.fit(train_data=train_dataiter, eval_data=val_dataiter, optimizer_params={'learning_rate':0.01, 'momentum': 0.9}, num_epoch=10) """ assert num_epoch is not None, 'please specify number of epochs' self.bind(data_shapes=train_data.provide_data, label_shapes=train_data.provide_label, for_training=True, force_rebind=force_rebind) if monitor is not None: self.install_monitor(monitor) self.init_params(initializer=initializer, arg_params=arg_params, aux_params=aux_params, allow_missing=allow_missing, force_init=force_init) self.init_optimizer(kvstore=kvstore, optimizer=optimizer, optimizer_params=optimizer_params) if validation_metric is None: validation_metric = eval_metric if not isinstance(eval_metric, metric.EvalMetric): eval_metric = metric.create(eval_metric) ################################################################################ # training loop ################################################################################ for epoch in range(begin_epoch, num_epoch): tic = time.time() eval_metric.reset() for nbatch, data_batch in enumerate(train_data): if monitor is not None: monitor.tic() self.forward_backward(data_batch) self.update() self.update_metric(eval_metric, data_batch.label) if monitor is not None: monitor.toc_print() if batch_end_callback is not None: batch_end_params = BatchEndParam(epoch=epoch, nbatch=nbatch, eval_metric=eval_metric, locals=locals()) for callback in _as_list(batch_end_callback): callback(batch_end_params) # one epoch of training is finished for name, val in eval_metric.get_name_value(): self.logger.info('Epoch[%d] Train-%s=%f', epoch, name, val) toc = time.time() self.logger.info('Epoch[%d] Time cost=%.3f', epoch, (toc-tic)) # sync aux params across devices arg_params, aux_params = self.get_params() self.set_params(arg_params, aux_params) if epoch_end_callback is not None: for callback in _as_list(epoch_end_callback): callback(epoch, self.symbol, arg_params, aux_params) #---------------------------------------- # evaluation on validation set if eval_data: res = self.score(eval_data, validation_metric, score_end_callback=eval_end_callback, batch_end_callback=eval_batch_end_callback, epoch=epoch) #TODO: pull this into default for name, val in res: self.logger.info('Epoch[%d] Validation-%s=%f', epoch, name, val) # end of 1 epoch, reset the data-iter for another epoch train_data.reset() def forward(self, data_batch, is_train=None): assert self.binded and self.params_initialized # get current_shapes if self._curr_module.label_shapes is not None: current_shapes = [dict(self._curr_module.data_shapes[i] + self._curr_module.label_shapes[i]) for i in xrange(len(self._context))] else: current_shapes = [dict(self._curr_module.data_shapes[i]) for i in xrange(len(self._context))] # get input_shapes if is_train: input_shapes = [dict(data_batch.provide_data[i] + data_batch.provide_label[i]) for i in xrange(len(self._context))] else: input_shapes = [dict(data_batch.provide_data[i]) for i in xrange(len(data_batch.provide_data))] # decide if shape changed shape_changed = len(current_shapes) != len(input_shapes) for pre, cur in zip(current_shapes, input_shapes): for k, v in pre.items(): if v != cur[k]: shape_changed = True if shape_changed: # self._curr_module.reshape(data_batch.provide_data, data_batch.provide_label) module = Module(self._symbol, self._data_names, self._label_names, logger=self.logger, context=[self._context[i] for i in xrange(len(data_batch.provide_data))], work_load_list=self._work_load_list, fixed_param_names=self._fixed_param_names) module.bind(data_batch.provide_data, data_batch.provide_label, self._curr_module.for_training, self._curr_module.inputs_need_grad, force_rebind=False, shared_module=self._curr_module) self._curr_module = module self._curr_module.forward(data_batch, is_train=is_train) def backward(self, out_grads=None): assert self.binded and self.params_initialized self._curr_module.backward(out_grads=out_grads) def update(self): assert self.binded and self.params_initialized and self.optimizer_initialized self._curr_module.update() def get_outputs(self, merge_multi_context=True): assert self.binded and self.params_initialized return self._curr_module.get_outputs(merge_multi_context=merge_multi_context) def get_input_grads(self, merge_multi_context=True): assert self.binded and self.params_initialized and self.inputs_need_grad return self._curr_module.get_input_grads(merge_multi_context=merge_multi_context) def update_metric(self, eval_metric, labels): assert self.binded and self.params_initialized self._curr_module.update_metric(eval_metric, labels) def install_monitor(self, mon): """ Install monitor on all executors """ assert self.binded self._curr_module.install_monitor(mon) ================================================ FILE: rfcn/core/rcnn.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- """ Fast R-CNN: data = {'data': [num_images, c, h, w], 'rois': [num_rois, 5]} label = {'label': [num_rois], 'bbox_target': [num_rois, 4 * num_classes], 'bbox_weight': [num_rois, 4 * num_classes]} roidb extended format [image_index] ['image', 'height', 'width', 'flipped', 'boxes', 'gt_classes', 'gt_overlaps', 'max_classes', 'max_overlaps', 'bbox_targets'] """ import numpy as np import numpy.random as npr from utils.image import get_image, tensor_vstack from bbox.bbox_transform import bbox_overlaps, bbox_transform from bbox.bbox_regression import expand_bbox_regression_targets def get_rcnn_testbatch(roidb, cfg): """ return a dict of testbatch :param roidb: ['image', 'flipped'] + ['boxes'] :return: data, label, im_info """ # assert len(roidb) == 1, 'Single batch only' imgs, roidb = get_image(roidb, cfg) im_array = imgs im_info = [np.array([roidb[i]['im_info']], dtype=np.float32) for i in range(len(roidb))] im_rois = [roidb[i]['boxes'] for i in range(len(roidb))] rois = im_rois rois_array = [np.hstack((0 * np.ones((rois[i].shape[0], 1)), rois[i])) for i in range(len(rois))] data = [{'data': im_array[i], 'rois': rois_array[i]} for i in range(len(roidb))] label = {} return data, label, im_info def get_rcnn_batch(roidb, cfg): """ return a dict of multiple images :param roidb: a list of dict, whose length controls batch size ['images', 'flipped'] + ['gt_boxes', 'boxes', 'gt_overlap'] => ['bbox_targets'] :return: data, label """ num_images = len(roidb) imgs, roidb = get_image(roidb, cfg) im_array = tensor_vstack(imgs) assert cfg.TRAIN.BATCH_ROIS == -1 or cfg.TRAIN.BATCH_ROIS % cfg.TRAIN.BATCH_IMAGES == 0, \ 'BATCHIMAGES {} must divide BATCH_ROIS {}'.format(cfg.TRAIN.BATCH_IMAGES, cfg.TRAIN.BATCH_ROIS) if cfg.TRAIN.BATCH_ROIS == -1: rois_per_image = np.sum([iroidb['boxes'].shape[0] for iroidb in roidb]) fg_rois_per_image = rois_per_image else: rois_per_image = cfg.TRAIN.BATCH_ROIS / cfg.TRAIN.BATCH_IMAGES fg_rois_per_image = np.round(cfg.TRAIN.FG_FRACTION * rois_per_image).astype(int) rois_array = list() labels_array = list() bbox_targets_array = list() bbox_weights_array = list() for im_i in range(num_images): roi_rec = roidb[im_i] # infer num_classes from gt_overlaps num_classes = roi_rec['gt_overlaps'].shape[1] # label = class RoI has max overlap with rois = roi_rec['boxes'] labels = roi_rec['max_classes'] overlaps = roi_rec['max_overlaps'] bbox_targets = roi_rec['bbox_targets'] im_rois, labels, bbox_targets, bbox_weights = \ sample_rois(rois, fg_rois_per_image, rois_per_image, num_classes, cfg, labels, overlaps, bbox_targets) # project im_rois # do not round roi rois = im_rois batch_index = im_i * np.ones((rois.shape[0], 1)) rois_array_this_image = np.hstack((batch_index, rois)) rois_array.append(rois_array_this_image) # add labels labels_array.append(labels) bbox_targets_array.append(bbox_targets) bbox_weights_array.append(bbox_weights) rois_array = np.array(rois_array) labels_array = np.array(labels_array) bbox_targets_array = np.array(bbox_targets_array) bbox_weights_array = np.array(bbox_weights_array) data = {'data': im_array, 'rois': rois_array} label = {'label': labels_array, 'bbox_target': bbox_targets_array, 'bbox_weight': bbox_weights_array} return data, label def sample_rois(rois, fg_rois_per_image, rois_per_image, num_classes, cfg, labels=None, overlaps=None, bbox_targets=None, gt_boxes=None): """ generate random sample of ROIs comprising foreground and background examples :param rois: all_rois [n, 4]; e2e: [n, 5] with batch_index :param fg_rois_per_image: foreground roi number :param rois_per_image: total roi number :param num_classes: number of classes :param labels: maybe precomputed :param overlaps: maybe precomputed (max_overlaps) :param bbox_targets: maybe precomputed :param gt_boxes: optional for e2e [n, 5] (x1, y1, x2, y2, cls) :return: (labels, rois, bbox_targets, bbox_weights) """ if labels is None: overlaps = bbox_overlaps(rois[:, 1:].astype(np.float), gt_boxes[:, :4].astype(np.float)) gt_assignment = overlaps.argmax(axis=1) overlaps = overlaps.max(axis=1) labels = gt_boxes[gt_assignment, 4] # foreground RoI with FG_THRESH overlap fg_indexes = np.where(overlaps >= cfg.TRAIN.FG_THRESH)[0] # guard against the case when an image has fewer than fg_rois_per_image foreground RoIs fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_indexes.size) # Sample foreground regions without replacement if len(fg_indexes) > fg_rois_per_this_image: fg_indexes = npr.choice(fg_indexes, size=fg_rois_per_this_image, replace=False) # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_indexes = np.where((overlaps < cfg.TRAIN.BG_THRESH_HI) & (overlaps >= cfg.TRAIN.BG_THRESH_LO))[0] # Compute number of background RoIs to take from this image (guarding against there being fewer than desired) bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_indexes.size) # Sample foreground regions without replacement if len(bg_indexes) > bg_rois_per_this_image: bg_indexes = npr.choice(bg_indexes, size=bg_rois_per_this_image, replace=False) # indexes selected keep_indexes = np.append(fg_indexes, bg_indexes) # pad more to ensure a fixed minibatch size while keep_indexes.shape[0] < rois_per_image: gap = np.minimum(len(rois), rois_per_image - keep_indexes.shape[0]) gap_indexes = npr.choice(range(len(rois)), size=gap, replace=False) keep_indexes = np.append(keep_indexes, gap_indexes) # select labels labels = labels[keep_indexes] # set labels of bg_rois to be 0 labels[fg_rois_per_this_image:] = 0 rois = rois[keep_indexes] # load or compute bbox_target if bbox_targets is not None: bbox_target_data = bbox_targets[keep_indexes, :] else: targets = bbox_transform(rois[:, 1:], gt_boxes[gt_assignment[keep_indexes], :4]) if cfg.TRAIN.BBOX_NORMALIZATION_PRECOMPUTED: targets = ((targets - np.array(cfg.TRAIN.BBOX_MEANS)) / np.array(cfg.TRAIN.BBOX_STDS)) bbox_target_data = np.hstack((labels[:, np.newaxis], targets)) bbox_targets, bbox_weights = \ expand_bbox_regression_targets(bbox_target_data, num_classes, cfg) return rois, labels, bbox_targets, bbox_weights ================================================ FILE: rfcn/core/tester.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- import cPickle import os import time import mxnet as mx import numpy as np from module import MutableModule from utils import image from bbox.bbox_transform import bbox_pred, clip_boxes from nms.nms import py_nms_wrapper, cpu_nms_wrapper, gpu_nms_wrapper from utils.PrefetchingIter import PrefetchingIter class Predictor(object): def __init__(self, symbol, data_names, label_names, context=mx.cpu(), max_data_shapes=None, provide_data=None, provide_label=None, arg_params=None, aux_params=None): self._mod = MutableModule(symbol, data_names, label_names, context=context, max_data_shapes=max_data_shapes) self._mod.bind(provide_data, provide_label, for_training=False) self._mod.init_params(arg_params=arg_params, aux_params=aux_params) def predict(self, data_batch): self._mod.forward(data_batch) # [dict(zip(self._mod.output_names, _)) for _ in zip(*self._mod.get_outputs(merge_multi_context=False))] return [dict(zip(self._mod.output_names, _)) for _ in zip(*self._mod.get_outputs(merge_multi_context=False))] def im_proposal(predictor, data_batch, data_names, scales): output_all = predictor.predict(data_batch) data_dict_all = [dict(zip(data_names, data_batch.data[i])) for i in xrange(len(data_batch.data))] scores_all = [] boxes_all = [] for output, data_dict, scale in zip(output_all, data_dict_all, scales): # drop the batch index boxes = output['rois_output'].asnumpy()[:, 1:] scores = output['rois_score'].asnumpy() # transform to original scale boxes = boxes / scale scores_all.append(scores) boxes_all.append(boxes) return scores_all, boxes_all, data_dict_all def generate_proposals(predictor, test_data, imdb, cfg, vis=False, thresh=0.): """ Generate detections results using RPN. :param predictor: Predictor :param test_data: data iterator, must be non-shuffled :param imdb: image database :param vis: controls visualization :param thresh: thresh for valid detections :return: list of detected boxes """ assert vis or not test_data.shuffle data_names = [k[0] for k in test_data.provide_data[0]] if not isinstance(test_data, PrefetchingIter): test_data = PrefetchingIter(test_data) idx = 0 t = time.clock() imdb_boxes = list() original_boxes = list() for im_info, data_batch in test_data: t1 = time.clock() - t t = time.clock() scales = [iim_info[0, 2] for iim_info in im_info] scores_all, boxes_all, data_dict_all = im_proposal(predictor, data_batch, data_names, scales) t2 = time.clock() - t t = time.clock() for delta, (scores, boxes, data_dict, scale) in enumerate(zip(scores_all, boxes_all, data_dict_all, scales)): # assemble proposals dets = np.hstack((boxes, scores)) original_boxes.append(dets) # filter proposals keep = np.where(dets[:, 4:] > thresh)[0] dets = dets[keep, :] imdb_boxes.append(dets) if vis: vis_all_detection(data_dict['data'].asnumpy(), [dets], ['obj'], scale, cfg) print 'generating %d/%d' % (idx + 1, imdb.num_images), 'proposal %d' % (dets.shape[0]), \ 'data %.4fs net %.4fs' % (t1, t2 / test_data.batch_size) idx += 1 assert len(imdb_boxes) == imdb.num_images, 'calculations not complete' # save results rpn_folder = os.path.join(imdb.result_path, 'rpn_data') if not os.path.exists(rpn_folder): os.mkdir(rpn_folder) rpn_file = os.path.join(rpn_folder, imdb.name + '_rpn.pkl') with open(rpn_file, 'wb') as f: cPickle.dump(imdb_boxes, f, cPickle.HIGHEST_PROTOCOL) if thresh > 0: full_rpn_file = os.path.join(rpn_folder, imdb.name + '_full_rpn.pkl') with open(full_rpn_file, 'wb') as f: cPickle.dump(original_boxes, f, cPickle.HIGHEST_PROTOCOL) print 'wrote rpn proposals to {}'.format(rpn_file) return imdb_boxes def im_detect(predictor, data_batch, data_names, scales, cfg): output_all = predictor.predict(data_batch) data_dict_all = [dict(zip(data_names, idata)) for idata in data_batch.data] scores_all = [] pred_boxes_all = [] for output, data_dict, scale in zip(output_all, data_dict_all, scales): if cfg.TEST.HAS_RPN: rois = output['rois_output'].asnumpy()[:, 1:] else: rois = data_dict['rois'].asnumpy().reshape((-1, 5))[:, 1:] im_shape = data_dict['data'].shape # save output scores = output['cls_prob_reshape_output'].asnumpy()[0] bbox_deltas = output['bbox_pred_reshape_output'].asnumpy()[0] # post processing pred_boxes = bbox_pred(rois, bbox_deltas) pred_boxes = clip_boxes(pred_boxes, im_shape[-2:]) # we used scaled image & roi to train, so it is necessary to transform them back pred_boxes = pred_boxes / scale scores_all.append(scores) pred_boxes_all.append(pred_boxes) return scores_all, pred_boxes_all, data_dict_all def im_batch_detect(predictor, data_batch, data_names, scales, cfg): output_all = predictor.predict(data_batch) data_dict_all = [dict(zip(data_names, data_batch.data[i])) for i in xrange(len(data_batch.data))] scores_all = [] pred_boxes_all = [] for output, data_dict, scale in zip(output_all, data_dict_all, scales): im_infos = data_dict['im_info'].asnumpy() # save output scores = output['cls_prob_reshape_output'].asnumpy()[0] bbox_deltas = output['bbox_pred_reshape_output'].asnumpy()[0] rois = output['rois_output'].asnumpy() for im_idx in xrange(im_infos.shape[0]): bb_idxs = np.where(rois[:,0] == im_idx)[0] im_shape = im_infos[im_idx, :2].astype(np.int) # post processing pred_boxes = bbox_pred(rois[bb_idxs, 1:], bbox_deltas[bb_idxs, :]) pred_boxes = clip_boxes(pred_boxes, im_shape) # we used scaled image & roi to train, so it is necessary to transform them back pred_boxes = pred_boxes / scale[im_idx] scores_all.append(scores[bb_idxs, :]) pred_boxes_all.append(pred_boxes) return scores_all, pred_boxes_all, data_dict_all def pred_eval(predictor, test_data, imdb, cfg, vis=False, thresh=1e-3, logger=None, ignore_cache=True): """ wrapper for calculating offline validation for faster data analysis in this example, all threshold are set by hand :param predictor: Predictor :param test_data: data iterator, must be non-shuffle :param imdb: image database :param vis: controls visualization :param thresh: valid detection threshold :return: """ det_file = os.path.join(imdb.result_path, imdb.name + '_detections.pkl') if os.path.exists(det_file) and not ignore_cache: with open(det_file, 'rb') as fid: all_boxes = cPickle.load(fid) info_str = imdb.evaluate_detections(all_boxes) if logger: logger.info('evaluate detections: \n{}'.format(info_str)) return assert vis or not test_data.shuffle data_names = [k[0] for k in test_data.provide_data[0]] if not isinstance(test_data, PrefetchingIter): test_data = PrefetchingIter(test_data) nms = py_nms_wrapper(cfg.TEST.NMS) # limit detections to max_per_image over all classes max_per_image = cfg.TEST.max_per_image num_images = imdb.num_images # all detections are collected into: # all_boxes[cls][image] = N x 5 array of detections in # (x1, y1, x2, y2, score) all_boxes = [[[] for _ in range(num_images)] for _ in range(imdb.num_classes)] idx = 0 data_time, net_time, post_time = 0.0, 0.0, 0.0 t = time.clock() for im_info, data_batch in test_data: t1 = time.clock() - t t = time.clock() scales = [iim_info[0, 2] for iim_info in im_info] scores_all, boxes_all, data_dict_all = im_detect(predictor, data_batch, data_names, scales, cfg) t2 = time.clock() - t t = time.clock() for delta, (scores, boxes, data_dict) in enumerate(zip(scores_all, boxes_all, data_dict_all)): for j in range(1, imdb.num_classes): indexes = np.where(scores[:, j] > thresh)[0] cls_scores = scores[indexes, j, np.newaxis] cls_boxes = boxes[indexes, 4:8] if cfg.CLASS_AGNOSTIC else boxes[indexes, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) all_boxes[j][idx+delta] = cls_dets[keep, :] if max_per_image > 0: image_scores = np.hstack([all_boxes[j][idx+delta][:, -1] for j in range(1, imdb.num_classes)]) if len(image_scores) > max_per_image: image_thresh = np.sort(image_scores)[-max_per_image] for j in range(1, imdb.num_classes): keep = np.where(all_boxes[j][idx+delta][:, -1] >= image_thresh)[0] all_boxes[j][idx+delta] = all_boxes[j][idx+delta][keep, :] if vis: boxes_this_image = [[]] + [all_boxes[j][idx+delta] for j in range(1, imdb.num_classes)] vis_all_detection(data_dict['data'].asnumpy(), boxes_this_image, imdb.classes, scales[delta], cfg) idx += test_data.batch_size t3 = time.clock() - t t = time.clock() data_time += t1 net_time += t2 post_time += t3 print 'testing {}/{} data {:.4f}s net {:.4f}s post {:.4f}s'.format(idx, imdb.num_images, data_time / idx * test_data.batch_size, net_time / idx * test_data.batch_size, post_time / idx * test_data.batch_size) if logger: logger.info('testing {}/{} data {:.4f}s net {:.4f}s post {:.4f}s'.format(idx, imdb.num_images, data_time / idx * test_data.batch_size, net_time / idx * test_data.batch_size, post_time / idx * test_data.batch_size)) with open(det_file, 'wb') as f: cPickle.dump(all_boxes, f, protocol=cPickle.HIGHEST_PROTOCOL) info_str = imdb.evaluate_detections(all_boxes) if logger: logger.info('evaluate detections: \n{}'.format(info_str)) def vis_all_detection(im_array, detections, class_names, scale, cfg, threshold=1e-3): """ visualize all detections in one image :param im_array: [b=1 c h w] in rgb :param detections: [ numpy.ndarray([[x1 y1 x2 y2 score]]) for j in classes ] :param class_names: list of names in imdb :param scale: visualize the scaled image :return: """ import matplotlib.pyplot as plt import random im = image.transform_inverse(im_array, cfg.network.PIXEL_MEANS) plt.imshow(im) for j, name in enumerate(class_names): if name == '__background__': continue color = (random.random(), random.random(), random.random()) # generate a random color dets = detections[j] for det in dets: bbox = det[:4] * scale score = det[-1] if score < threshold: continue rect = plt.Rectangle((bbox[0], bbox[1]), bbox[2] - bbox[0], bbox[3] - bbox[1], fill=False, edgecolor=color, linewidth=3.5) plt.gca().add_patch(rect) plt.gca().text(bbox[0], bbox[1] - 2, '{:s} {:.3f}'.format(name, score), bbox=dict(facecolor=color, alpha=0.5), fontsize=12, color='white') plt.show() def draw_all_detection(im_array, detections, class_names, scale, cfg, threshold=1e-1): """ visualize all detections in one image :param im_array: [b=1 c h w] in rgb :param detections: [ numpy.ndarray([[x1 y1 x2 y2 score]]) for j in classes ] :param class_names: list of names in imdb :param scale: visualize the scaled image :return: """ import cv2 import random color_white = (255, 255, 255) im = image.transform_inverse(im_array, cfg.network.PIXEL_MEANS) # change to bgr im = cv2.cvtColor(im, cv2.COLOR_RGB2BGR) for j, name in enumerate(class_names): if name == '__background__': continue color = (random.randint(0, 256), random.randint(0, 256), random.randint(0, 256)) # generate a random color dets = detections[j] for det in dets: bbox = det[:4] * scale score = det[-1] if score < threshold: continue bbox = map(int, bbox) cv2.rectangle(im, (bbox[0], bbox[1]), (bbox[2], bbox[3]), color=color, thickness=2) cv2.putText(im, '%s %.3f' % (class_names[j], score), (bbox[0], bbox[1] + 10), color=color_white, fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.5) return im ================================================ FILE: rfcn/demo.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Xizhou Zhu, Yi Li, Haochen Zhang # -------------------------------------------------------- import _init_paths import argparse import os import glob import sys import logging import pprint import cv2 from config.config import config, update_config from utils.image import resize, transform import numpy as np # get config os.environ['PYTHONUNBUFFERED'] = '1' os.environ['MXNET_CUDNN_AUTOTUNE_DEFAULT'] = '0' os.environ['MXNET_ENABLE_GPU_P2P'] = '0' cur_path = os.path.abspath(os.path.dirname(__file__)) update_config(cur_path + '/../experiments/rfcn/cfgs/rfcn_vid_demo.yaml') sys.path.insert(0, os.path.join(cur_path, '../external/mxnet', config.MXNET_VERSION)) import mxnet as mx from core.tester import im_detect, Predictor from symbols import * from utils.load_model import load_param from utils.show_boxes import show_boxes, draw_boxes from utils.tictoc import tic, toc from nms.nms import py_nms_wrapper, cpu_nms_wrapper, gpu_nms_wrapper def parse_args(): parser = argparse.ArgumentParser(description='Show Deep Feature Flow demo') args = parser.parse_args() return args args = parse_args() def main(): # get symbol pprint.pprint(config) config.symbol = 'resnet_v1_101_rfcn' model = '/../model/rfcn_vid' sym_instance = eval(config.symbol + '.' + config.symbol)() sym = sym_instance.get_test_symbol(config) # set up class names num_classes = 31 classes = ['airplane', 'antelope', 'bear', 'bicycle', 'bird', 'bus', 'car', 'cattle', 'dog', 'domestic_cat', 'elephant', 'fox', 'giant_panda', 'hamster', 'horse', 'lion', 'lizard', 'monkey', 'motorcycle', 'rabbit', 'red_panda', 'sheep', 'snake', 'squirrel', 'tiger', 'train', 'turtle', 'watercraft', 'whale', 'zebra'] # load demo data image_names = glob.glob(cur_path + '/../demo/ILSVRC2015_val_00007010/*.JPEG') output_dir = cur_path + '/../demo/rfcn/' if not os.path.exists(output_dir): os.makedirs(output_dir) # data = [] for im_name in image_names: assert os.path.exists(im_name), ('%s does not exist'.format(im_name)) im = cv2.imread(im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] arg_params, aux_params = load_param(cur_path + model, 0, process=True) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) nms = gpu_nms_wrapper(config.TEST.NMS, 0) # warm up for j in xrange(2): data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[0])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))] scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config) # test time = 0 count = 0 for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))] tic() scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config) time += toc() count += 1 print 'testing {} {:.4f}s'.format(im_name, time/count) boxes = boxes[0].astype('f') scores = scores[0].astype('f') dets_nms = [] for j in range(1, scores.shape[1]): cls_scores = scores[:, j, np.newaxis] cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) cls_dets = cls_dets[keep, :] cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :] dets_nms.append(cls_dets) # visualize im = cv2.imread(im_name) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) # show_boxes(im, dets_nms, classes, 1) out_im = draw_boxes(im, dets_nms, classes, 1) _, filename = os.path.split(im_name) cv2.imwrite(output_dir + filename,out_im) print 'done' if __name__ == '__main__': main() ================================================ FILE: rfcn/demo_batch.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Xizhou Zhu, Yi Li, Haochen Zhang # -------------------------------------------------------- import _init_paths import argparse import os import glob import sys import logging import pprint import cv2 from config.config import config, update_config from utils.image import resize, transform import numpy as np # get config os.environ['PYTHONUNBUFFERED'] = '1' os.environ['MXNET_CUDNN_AUTOTUNE_DEFAULT'] = '0' os.environ['MXNET_ENABLE_GPU_P2P'] = '0' cur_path = os.path.abspath(os.path.dirname(__file__)) update_config(cur_path + '/../experiments/rfcn/cfgs/rfcn_vid_demo.yaml') sys.path.insert(0, os.path.join(cur_path, '../external/mxnet', config.MXNET_VERSION)) import mxnet as mx from core.tester import im_batch_detect, Predictor from symbols import * from utils.load_model import load_param from utils.show_boxes import show_boxes, draw_boxes from utils.tictoc import tic, toc from nms.nms import py_nms_wrapper, cpu_nms_wrapper, gpu_nms_wrapper def parse_args(): parser = argparse.ArgumentParser(description='Show Deep Feature Flow demo') args = parser.parse_args() return args args = parse_args() def main(): # get symbol pprint.pprint(config) config.symbol = 'resnet_v1_101_rfcn' model = '/../model/rfcn_vid' sym_instance = eval(config.symbol + '.' + config.symbol)() sym = sym_instance.get_test_symbol(config) # set up class names num_classes = 31 classes = ['airplane', 'antelope', 'bear', 'bicycle', 'bird', 'bus', 'car', 'cattle', 'dog', 'domestic_cat', 'elephant', 'fox', 'giant_panda', 'hamster', 'horse', 'lion', 'lizard', 'monkey', 'motorcycle', 'rabbit', 'red_panda', 'sheep', 'snake', 'squirrel', 'tiger', 'train', 'turtle', 'watercraft', 'whale', 'zebra'] # load demo data image_names = glob.glob(cur_path + '/../demo/ILSVRC2015_val_00007010/*.JPEG') output_dir = cur_path + '/../demo/rfcn_batch/' if not os.path.exists(output_dir): os.makedirs(output_dir) batch_size = 10 # data = [] cur_im_tensor = [] im_info_tensor = [] image_names_list = [] image_names_batch = [] for idx, im_name in enumerate(image_names): assert os.path.exists(im_name), ('%s does not exist'.format(im_name)) im = cv2.imread(im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) cur_im_tensor.append(im_tensor) im_info_tensor.append(im_info) image_names_batch.append(im_name) if (idx+1) % batch_size == 0 or idx == len(image_names) - 1: data.append({'data': np.concatenate(cur_im_tensor), 'im_info': np.concatenate(im_info_tensor)}) cur_im_tensor = [] im_info_tensor = [] image_names_list.append(image_names_batch) image_names_batch = [] # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (batch_size, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] arg_params, aux_params = load_param(cur_path + model, 0, process=True) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) nms = gpu_nms_wrapper(config.TEST.NMS, 0) # warm up for j in xrange(1): data_batch = mx.io.DataBatch(data=[data[j]], label=[], pad=0, index=0, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[j])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[:, 2] for i in xrange(len(data_batch.data))] scores_all, boxes_all, data_dict = im_batch_detect(predictor, data_batch, data_names, scales, config) print "warmup done" # test time = 0 count = 0 for idx, im_names in enumerate(image_names_list): data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[:, 2] for i in xrange(len(data_batch.data))] tic() scores_all, boxes_all, data_dict = im_batch_detect(predictor, data_batch, data_names, scales, config) time += toc() count += len(scores_all) print 'testing {} {:.4f}s x {:d}'.format(im_names[0], time/count, len(scores_all)) for batch_idx in xrange(len(scores_all)): boxes = boxes_all[batch_idx].astype('f') scores = scores_all[batch_idx].astype('f') dets_nms = [] for j in range(1, scores.shape[1]): cls_scores = scores[:, j, np.newaxis] cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) cls_dets = cls_dets[keep, :] cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :] dets_nms.append(cls_dets) # visualize im = cv2.imread(im_names[batch_idx]) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) # show_boxes(im, dets_nms, classes, 1) out_im = draw_boxes(im, dets_nms, classes, 1) _, filename = os.path.split(im_names[batch_idx]) cv2.imwrite(output_dir + filename,out_im) print 'done' if __name__ == '__main__': main() ================================================ FILE: rfcn/function/__init__.py ================================================ ================================================ FILE: rfcn/function/test_rcnn.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- import argparse import pprint import logging import time import os import mxnet as mx from symbols import * from dataset import * from core.loader import TestLoader from core.tester import Predictor, pred_eval from utils.load_model import load_param def test_rcnn(cfg, dataset, image_set, root_path, dataset_path, ctx, prefix, epoch, vis, ignore_cache, shuffle, has_rpn, proposal, thresh, logger=None, output_path=None): if not logger: assert False, 'require a logger' # print cfg pprint.pprint(cfg) logger.info('testing cfg:{}\n'.format(pprint.pformat(cfg))) # load symbol and testing data sym_instance = eval(cfg.symbol + '.' + cfg.symbol)() sym = sym_instance.get_test_symbol(cfg) imdb = eval(dataset)(image_set, root_path, dataset_path, result_path=output_path) roidb = imdb.gt_roidb() # get test data iter test_data = TestLoader(roidb, cfg, batch_size=len(ctx), shuffle=shuffle, has_rpn=has_rpn) # load model arg_params, aux_params = load_param(prefix, epoch, process=True) # infer shape data_shape_dict = dict(test_data.provide_data_single) sym_instance.infer_shape(data_shape_dict) sym_instance.check_parameter_shapes(arg_params, aux_params, data_shape_dict, is_train=False) # decide maximum shape data_names = [k[0] for k in test_data.provide_data_single] label_names = None max_data_shape = [[('data', (1, 3, max([v[0] for v in cfg.SCALES]), max([v[1] for v in cfg.SCALES])))]] if not has_rpn: max_data_shape.append(('rois', (cfg.TEST.PROPOSAL_POST_NMS_TOP_N + 30, 5))) # create predictor predictor = Predictor(sym, data_names, label_names, context=ctx, max_data_shapes=max_data_shape, provide_data=test_data.provide_data, provide_label=test_data.provide_label, arg_params=arg_params, aux_params=aux_params) # start detection pred_eval(predictor, test_data, imdb, cfg, vis=vis, ignore_cache=ignore_cache, thresh=thresh, logger=logger) ================================================ FILE: rfcn/function/test_rpn.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- import argparse import pprint import logging import mxnet as mx from symbols import * from dataset import * from core.loader import TestLoader from core.tester import Predictor, generate_proposals from utils.load_model import load_param def test_rpn(cfg, dataset, image_set, root_path, dataset_path, ctx, prefix, epoch, vis, shuffle, thresh, logger=None, output_path=None): # set up logger if not logger: logging.basicConfig() logger = logging.getLogger() logger.setLevel(logging.INFO) # rpn generate proposal cfg cfg.TEST.HAS_RPN = True # print cfg pprint.pprint(cfg) logger.info('testing rpn cfg:{}\n'.format(pprint.pformat(cfg))) # load symbol sym_instance = eval(cfg.symbol + '.' + cfg.symbol)() sym = sym_instance.get_symbol_rpn(cfg, is_train=False) # load dataset and prepare imdb for training imdb = eval(dataset)(image_set, root_path, dataset_path, result_path=output_path) roidb = imdb.gt_roidb() test_data = TestLoader(roidb, cfg, batch_size=len(ctx), shuffle=shuffle, has_rpn=True) # load model arg_params, aux_params = load_param(prefix, epoch) # infer shape data_shape_dict = dict(test_data.provide_data_single) sym_instance.infer_shape(data_shape_dict) # check parameters sym_instance.check_parameter_shapes(arg_params, aux_params, data_shape_dict, is_train=False) # decide maximum shape data_names = [k[0] for k in test_data.provide_data[0]] label_names = None if test_data.provide_label[0] is None else [k[0] for k in test_data.provide_label[0]] max_data_shape = [[('data', (1, 3, max([v[0] for v in cfg.SCALES]), max([v[1] for v in cfg.SCALES])))]] # create predictor predictor = Predictor(sym, data_names, label_names, context=ctx, max_data_shapes=max_data_shape, provide_data=test_data.provide_data, provide_label=test_data.provide_label, arg_params=arg_params, aux_params=aux_params) # start testing imdb_boxes = generate_proposals(predictor, test_data, imdb, cfg, vis=vis, thresh=thresh) all_log_info = imdb.evaluate_recall(roidb, candidate_boxes=imdb_boxes) logger.info(all_log_info) ================================================ FILE: rfcn/function/train_rcnn.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- import argparse import logging import pprint import os import mxnet as mx from symbols import * from core import callback, metric from core.loader import ROIIter from core.module import MutableModule from bbox.bbox_regression import add_bbox_regression_targets from utils.load_data import load_proposal_roidb, merge_roidb, filter_roidb from utils.load_model import load_param from utils.PrefetchingIter import PrefetchingIter from utils.lr_scheduler import WarmupMultiFactorScheduler def train_rcnn(cfg, dataset, image_set, root_path, dataset_path, frequent, kvstore, flip, shuffle, resume, ctx, pretrained, epoch, prefix, begin_epoch, end_epoch, train_shared, lr, lr_step, proposal, logger=None, output_path=None): # set up logger if not logger: logging.basicConfig() logger = logging.getLogger() logger.setLevel(logging.INFO) # load symbol sym_instance = eval(cfg.symbol + '.' + cfg.symbol)() sym = sym_instance.get_symbol_rfcn(cfg, is_train=True) # setup multi-gpu batch_size = len(ctx) input_batch_size = cfg.TRAIN.BATCH_IMAGES * batch_size # print cfg pprint.pprint(cfg) logger.info('training rcnn cfg:{}\n'.format(pprint.pformat(cfg))) # load dataset and prepare imdb for training image_sets = [iset for iset in image_set.split('+')] roidbs = [load_proposal_roidb(dataset, image_set, root_path, dataset_path, proposal=proposal, append_gt=True, flip=flip, result_path=output_path) for image_set in image_sets] roidb = merge_roidb(roidbs) roidb = filter_roidb(roidb, cfg) means, stds = add_bbox_regression_targets(roidb, cfg) # load training data train_data = ROIIter(roidb, cfg, batch_size=input_batch_size, shuffle=shuffle, ctx=ctx, aspect_grouping=cfg.TRAIN.ASPECT_GROUPING) # infer max shape max_data_shape = [('data', (cfg.TRAIN.BATCH_IMAGES, 3, max([v[0] for v in cfg.SCALES]), max([v[1] for v in cfg.SCALES])))] # infer shape data_shape_dict = dict(train_data.provide_data_single + train_data.provide_label_single) sym_instance.infer_shape(data_shape_dict) # load and initialize params if resume: print('continue training from ', begin_epoch) arg_params, aux_params = load_param(prefix, begin_epoch, convert=True) else: arg_params, aux_params = load_param(pretrained, epoch, convert=True) sym_instance.init_weight_rfcn(cfg, arg_params, aux_params) # check parameter shapes sym_instance.check_parameter_shapes(arg_params, aux_params, data_shape_dict) # prepare training # create solver data_names = [k[0] for k in train_data.provide_data_single] label_names = [k[0] for k in train_data.provide_label_single] if train_shared: fixed_param_prefix = cfg.network.FIXED_PARAMS_SHARED else: fixed_param_prefix = cfg.network.FIXED_PARAMS mod = MutableModule(sym, data_names=data_names, label_names=label_names, logger=logger, context=ctx, max_data_shapes=[max_data_shape for _ in range(batch_size)], fixed_param_prefix=fixed_param_prefix) if cfg.TRAIN.RESUME: mod._preload_opt_states = '%s-%04d.states'%(prefix, begin_epoch) # decide training params # metric eval_metric = metric.RCNNAccMetric(cfg) cls_metric = metric.RCNNLogLossMetric(cfg) bbox_metric = metric.RCNNL1LossMetric(cfg) eval_metrics = mx.metric.CompositeEvalMetric() for child_metric in [eval_metric, cls_metric, bbox_metric]: eval_metrics.add(child_metric) # callback batch_end_callback = callback.Speedometer(train_data.batch_size, frequent=frequent) epoch_end_callback = [mx.callback.module_checkpoint(mod, prefix, period=1, save_optimizer_states=True), callback.do_checkpoint(prefix, means, stds)] # decide learning rate base_lr = lr lr_factor = cfg.TRAIN.lr_factor lr_epoch = [float(epoch) for epoch in lr_step.split(',')] lr_epoch_diff = [epoch - begin_epoch for epoch in lr_epoch if epoch > begin_epoch] lr = base_lr * (lr_factor ** (len(lr_epoch) - len(lr_epoch_diff))) lr_iters = [int(epoch * len(roidb) / batch_size) for epoch in lr_epoch_diff] print('lr', lr, 'lr_epoch_diff', lr_epoch_diff, 'lr_iters', lr_iters) lr_scheduler = WarmupMultiFactorScheduler(lr_iters, lr_factor, cfg.TRAIN.warmup, cfg.TRAIN.warmup_lr, cfg.TRAIN.warmup_step) # optimizer optimizer_params = {'momentum': cfg.TRAIN.momentum, 'wd': cfg.TRAIN.wd, 'learning_rate': lr, 'lr_scheduler': lr_scheduler, 'rescale_grad': 1.0, 'clip_gradient': None} # train if not isinstance(train_data, PrefetchingIter): train_data = PrefetchingIter(train_data) mod.fit(train_data, eval_metric=eval_metrics, epoch_end_callback=epoch_end_callback, batch_end_callback=batch_end_callback, kvstore=kvstore, optimizer='sgd', optimizer_params=optimizer_params, arg_params=arg_params, aux_params=aux_params, begin_epoch=begin_epoch, num_epoch=end_epoch) ================================================ FILE: rfcn/function/train_rpn.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- import argparse import logging import pprint import mxnet as mx from symbols import * from core import callback, metric from core.loader import AnchorLoader from core.module import MutableModule from utils.load_data import load_gt_roidb, merge_roidb, filter_roidb from utils.load_model import load_param from utils.PrefetchingIter import PrefetchingIter from utils.lr_scheduler import WarmupMultiFactorScheduler def train_rpn(cfg, dataset, image_set, root_path, dataset_path, frequent, kvstore, flip, shuffle, resume, ctx, pretrained, epoch, prefix, begin_epoch, end_epoch, train_shared, lr, lr_step, logger=None, output_path=None): # set up logger if not logger: logging.basicConfig() logger = logging.getLogger() logger.setLevel(logging.INFO) # set up config cfg.TRAIN.BATCH_IMAGES = cfg.TRAIN.ALTERNATE.RPN_BATCH_IMAGES # load symbol sym_instance = eval(cfg.symbol + '.' + cfg.symbol)() sym = sym_instance.get_symbol_rpn(cfg, is_train=True) feat_sym = sym.get_internals()['rpn_cls_score_output'] # setup multi-gpu batch_size = len(ctx) input_batch_size = cfg.TRAIN.BATCH_IMAGES * batch_size # print cfg pprint.pprint(cfg) logger.info('training rpn cfg:{}\n'.format(pprint.pformat(cfg))) # load dataset and prepare imdb for training image_sets = [iset for iset in image_set.split('+')] roidbs = [load_gt_roidb(dataset, image_set, root_path, dataset_path, result_path=output_path, flip=flip) for image_set in image_sets] roidb = merge_roidb(roidbs) roidb = filter_roidb(roidb, cfg) # load training data train_data = AnchorLoader(feat_sym, roidb, cfg, batch_size=input_batch_size, shuffle=shuffle, ctx=ctx, feat_stride=cfg.network.RPN_FEAT_STRIDE, anchor_scales=cfg.network.ANCHOR_SCALES, anchor_ratios=cfg.network.ANCHOR_RATIOS, aspect_grouping=cfg.TRAIN.ASPECT_GROUPING) # infer max shape max_data_shape = [('data', (cfg.TRAIN.BATCH_IMAGES, 3, max([v[0] for v in cfg.SCALES]), max([v[1] for v in cfg.SCALES])))] max_data_shape, max_label_shape = train_data.infer_shape(max_data_shape) print('providing maximum shape', max_data_shape, max_label_shape) # infer shape data_shape_dict = dict(train_data.provide_data_single + train_data.provide_label_single) sym_instance.infer_shape(data_shape_dict) # load and initialize params if resume: print('continue training from ', begin_epoch) arg_params, aux_params = load_param(prefix, begin_epoch, convert=True) else: arg_params, aux_params = load_param(pretrained, epoch, convert=True) sym_instance.init_weight_rpn(cfg, arg_params, aux_params) # check parameter shapes sym_instance.check_parameter_shapes(arg_params, aux_params, data_shape_dict) # create solver data_names = [k[0] for k in train_data.provide_data_single] label_names = [k[0] for k in train_data.provide_label_single] if train_shared: fixed_param_prefix = cfg.network.FIXED_PARAMS_SHARED else: fixed_param_prefix = cfg.network.FIXED_PARAMS mod = MutableModule(sym, data_names=data_names, label_names=label_names, logger=logger, context=ctx, max_data_shapes=[max_data_shape for _ in xrange(batch_size)], max_label_shapes=[max_label_shape for _ in xrange(batch_size)], fixed_param_prefix=fixed_param_prefix) # decide training params # metric eval_metric = metric.RPNAccMetric() cls_metric = metric.RPNLogLossMetric() bbox_metric = metric.RPNL1LossMetric() eval_metrics = mx.metric.CompositeEvalMetric() for child_metric in [eval_metric, cls_metric, bbox_metric]: eval_metrics.add(child_metric) # callback batch_end_callback = callback.Speedometer(train_data.batch_size, frequent=frequent) # epoch_end_callback = mx.callback.do_checkpoint(prefix) epoch_end_callback = mx.callback.module_checkpoint(mod, prefix, period=1, save_optimizer_states=True) # decide learning rate base_lr = lr lr_factor = cfg.TRAIN.lr_factor lr_epoch = [int(epoch) for epoch in lr_step.split(',')] lr_epoch_diff = [epoch - begin_epoch for epoch in lr_epoch if epoch > begin_epoch] lr = base_lr * (lr_factor ** (len(lr_epoch) - len(lr_epoch_diff))) lr_iters = [int(epoch * len(roidb) / batch_size) for epoch in lr_epoch_diff] print('lr', lr, 'lr_epoch_diff', lr_epoch_diff, 'lr_iters', lr_iters) lr_scheduler = WarmupMultiFactorScheduler(lr_iters, lr_factor, cfg.TRAIN.warmup, cfg.TRAIN.warmup_lr, cfg.TRAIN.warmup_step) # optimizer optimizer_params = {'momentum': cfg.TRAIN.momentum, 'wd': cfg.TRAIN.wd, 'learning_rate': lr, 'lr_scheduler': lr_scheduler, 'rescale_grad': 1.0, 'clip_gradient': None} if not isinstance(train_data, PrefetchingIter): train_data = PrefetchingIter(train_data) # train mod.fit(train_data, eval_metric=eval_metrics, epoch_end_callback=epoch_end_callback, batch_end_callback=batch_end_callback, kvstore=kvstore, optimizer='sgd', optimizer_params=optimizer_params, arg_params=arg_params, aux_params=aux_params, begin_epoch=begin_epoch, num_epoch=end_epoch) ================================================ FILE: rfcn/operator_cxx/multi_proposal-inl.h ================================================ /*! * Copyright (c) 2015 by Contributors * Copyright (c) 2017 Microsoft * Licensed under The MIT License [see LICENSE for details] * \file multi_proposal-inl.h * \brief MultiProposal Operator * \author Piotr Teterwak, Bing Xu, Jian Guo, Xizhou Zhu */ #ifndef MXNET_OPERATOR_CONTRIB_PROPOSAL_INL_H_ #define MXNET_OPERATOR_CONTRIB_PROPOSAL_INL_H_ #include #include #include #include #include #include #include #include #include #include #include "../operator_common.h" #include "../mshadow_op.h" // extend NumericalParam namespace mxnet { namespace op { /*! * \brief structure for numerical tuple input * \tparam VType data type of param */ template struct NumericalParam { NumericalParam() {} explicit NumericalParam(VType *begin, VType *end) { int32_t size = static_cast(end - begin); info.resize(size); for (int i = 0; i < size; ++i) { info[i] = *(begin + i); } } inline size_t ndim() const { return info.size(); } std::vector info; }; template inline std::istream &operator>>(std::istream &is, NumericalParam ¶m) { while (true) { char ch = is.get(); if (ch == '(') break; if (!isspace(ch)) { is.setstate(std::ios::failbit); return is; } } VType idx; std::vector tmp; // deal with empty case size_t pos = is.tellg(); char ch = is.get(); if (ch == ')') { param.info = tmp; return is; } is.seekg(pos); // finish deal while (is >> idx) { tmp.push_back(idx); char ch; do { ch = is.get(); } while (isspace(ch)); if (ch == ',') { while (true) { ch = is.peek(); if (isspace(ch)) { is.get(); continue; } if (ch == ')') { is.get(); break; } break; } if (ch == ')') break; } else if (ch == ')') { break; } else { is.setstate(std::ios::failbit); return is; } } param.info = tmp; return is; } template inline std::ostream &operator<<(std::ostream &os, const NumericalParam ¶m) { os << '('; for (index_t i = 0; i < param.info.size(); ++i) { if (i != 0) os << ','; os << param.info[i]; } // python style tuple if (param.info.size() == 1) os << ','; os << ')'; return os; } } // namespace op } // namespace mxnet namespace mxnet { namespace op { namespace proposal { enum MultiProposalOpInputs {kClsProb, kBBoxPred, kImInfo}; enum MultiProposalOpOutputs {kOut, kScore}; enum MultiProposalForwardResource {kTempResource}; } // proposal struct MultiProposalParam : public dmlc::Parameter { int rpn_pre_nms_top_n; int rpn_post_nms_top_n; float threshold; int rpn_min_size; NumericalParam scales; NumericalParam ratios; int feature_stride; bool output_score; bool iou_loss; DMLC_DECLARE_PARAMETER(MultiProposalParam) { float tmp[] = {0, 0, 0, 0}; DMLC_DECLARE_FIELD(rpn_pre_nms_top_n).set_default(6000) .describe("Number of top scoring boxes to keep after applying NMS to RPN proposals"); DMLC_DECLARE_FIELD(rpn_post_nms_top_n).set_default(300) .describe("Overlap threshold used for non-maximum" "suppresion(suppress boxes with IoU >= this threshold"); DMLC_DECLARE_FIELD(threshold).set_default(0.7) .describe("NMS value, below which to suppress."); DMLC_DECLARE_FIELD(rpn_min_size).set_default(16) .describe("Minimum height or width in proposal"); tmp[0] = 4.0f; tmp[1] = 8.0f; tmp[2] = 16.0f; tmp[3] = 32.0f; DMLC_DECLARE_FIELD(scales).set_default(NumericalParam(tmp, tmp + 4)) .describe("Used to generate anchor windows by enumerating scales"); tmp[0] = 0.5f; tmp[1] = 1.0f; tmp[2] = 2.0f; DMLC_DECLARE_FIELD(ratios).set_default(NumericalParam(tmp, tmp + 3)) .describe("Used to generate anchor windows by enumerating ratios"); DMLC_DECLARE_FIELD(feature_stride).set_default(16) .describe("The size of the receptive field each unit in the convolution layer of the rpn," "for example the product of all stride's prior to this layer."); DMLC_DECLARE_FIELD(output_score).set_default(false) .describe("Add score to outputs"); DMLC_DECLARE_FIELD(iou_loss).set_default(false) .describe("Usage of IoU Loss"); } }; template Operator *CreateOp(MultiProposalParam param); #if DMLC_USE_CXX11 class MultiProposalProp : public OperatorProperty { public: void Init(const std::vector >& kwargs) override { param_.Init(kwargs); } std::map GetParams() const override { return param_.__DICT__(); } bool InferShape(std::vector *in_shape, std::vector *out_shape, std::vector *aux_shape) const override { using namespace mshadow; CHECK_EQ(in_shape->size(), 3) << "Input:[cls_prob, bbox_pred, im_info]"; const TShape &dshape = in_shape->at(proposal::kClsProb); if (dshape.ndim() == 0) return false; Shape<4> bbox_pred_shape; bbox_pred_shape = Shape4(dshape[0], dshape[1] * 2, dshape[2], dshape[3]); SHAPE_ASSIGN_CHECK(*in_shape, proposal::kBBoxPred, bbox_pred_shape); Shape<2> im_info_shape; im_info_shape = Shape2(dshape[0], 3); SHAPE_ASSIGN_CHECK(*in_shape, proposal::kImInfo, im_info_shape); out_shape->clear(); // output out_shape->push_back(Shape2(dshape[0] * param_.rpn_post_nms_top_n, 5)); // score out_shape->push_back(Shape2(dshape[0] * param_.rpn_post_nms_top_n, 1)); return true; } OperatorProperty* Copy() const override { auto ptr = new MultiProposalProp(); ptr->param_ = param_; return ptr; } std::string TypeString() const override { return "_contrib_MultiProposal"; } std::vector ForwardResource( const std::vector &in_shape) const override { return {ResourceRequest::kTempSpace}; } std::vector DeclareBackwardDependency( const std::vector &out_grad, const std::vector &in_data, const std::vector &out_data) const override { return {}; } int NumVisibleOutputs() const override { if (param_.output_score) { return 2; } else { return 1; } } int NumOutputs() const override { return 2; } std::vector ListArguments() const override { return {"cls_prob", "bbox_pred", "im_info"}; } std::vector ListOutputs() const override { return {"output", "score"}; } Operator* CreateOperator(Context ctx) const override; private: MultiProposalParam param_; }; // class MultiProposalProp #endif // DMLC_USE_CXX11 } // namespace op } // namespace mxnet //======================== // Anchor Generation Utils //======================== namespace mxnet { namespace op { namespace utils { inline void _MakeAnchor(float w, float h, float x_ctr, float y_ctr, std::vector *out_anchors) { out_anchors->push_back(x_ctr - 0.5f * (w - 1.0f)); out_anchors->push_back(y_ctr - 0.5f * (h - 1.0f)); out_anchors->push_back(x_ctr + 0.5f * (w - 1.0f)); out_anchors->push_back(y_ctr + 0.5f * (h - 1.0f)); out_anchors->push_back(0.0f); } inline void _Transform(float scale, float ratio, const std::vector& base_anchor, std::vector *out_anchors) { float w = base_anchor[2] - base_anchor[1] + 1.0f; float h = base_anchor[3] - base_anchor[1] + 1.0f; float x_ctr = base_anchor[0] + 0.5 * (w - 1.0f); float y_ctr = base_anchor[1] + 0.5 * (h - 1.0f); float size = w * h; float size_ratios = std::floor(size / ratio); float new_w = std::floor(std::sqrt(size_ratios) + 0.5f) * scale; float new_h = std::floor((new_w / scale * ratio) + 0.5f) * scale; _MakeAnchor(new_w, new_h, x_ctr, y_ctr, out_anchors); } // out_anchors must have shape (n, 5), where n is ratios.size() * scales.size() inline void GenerateAnchors(const std::vector& base_anchor, const std::vector& ratios, const std::vector& scales, std::vector *out_anchors) { for (size_t j = 0; j < ratios.size(); ++j) { for (size_t k = 0; k < scales.size(); ++k) { _Transform(scales[k], ratios[j], base_anchor, out_anchors); } } } } // namespace utils } // namespace op } // namespace mxnet #endif // MXNET_OPERATOR_CONTRIB_PROPOSAL_INL_H_ ================================================ FILE: rfcn/operator_cxx/multi_proposal.cc ================================================ /*! * Copyright (c) 2017 Microsoft * Licensed under The MIT License [see LICENSE for details] * \file multi_proposal.cc * \brief * \author Xizhou Zhu */ #include "./multi_proposal-inl.h" namespace mxnet { namespace op { template class MultiProposalOp : public Operator{ public: explicit MultiProposalOp(MultiProposalParam param) { this->param_ = param; } virtual void Forward(const OpContext &ctx, const std::vector &in_data, const std::vector &req, const std::vector &out_data, const std::vector &aux_states) { LOG(FATAL) << "not implemented"; } virtual void Backward(const OpContext &ctx, const std::vector &out_grad, const std::vector &in_data, const std::vector &out_data, const std::vector &req, const std::vector &in_grad, const std::vector &aux_states) { LOG(FATAL) << "not implemented"; } private: MultiProposalParam param_; }; // class MultiProposalOp template<> Operator *CreateOp(MultiProposalParam param) { return new MultiProposalOp(param); } Operator* MultiProposalProp::CreateOperator(Context ctx) const { DO_BIND_DISPATCH(CreateOp, param_); } DMLC_REGISTER_PARAMETER(MultiProposalParam); MXNET_REGISTER_OP_PROPERTY(_contrib_MultiProposal, MultiProposalProp) .describe("Generate region proposals via RPN") .add_argument("cls_score", "NDArray-or-Symbol", "Score of how likely proposal is object.") .add_argument("bbox_pred", "NDArray-or-Symbol", "BBox Predicted deltas from anchors for proposals") .add_argument("im_info", "NDArray-or-Symbol", "Image size and scale.") .add_arguments(MultiProposalParam::__FIELDS__()); } // namespace op } // namespace mxnet ================================================ FILE: rfcn/operator_cxx/multi_proposal.cu ================================================ /*! * Copyright (c) 2015 by Contributors * Copyright (c) 2017 Microsoft * Licensed under The MIT License [see LICENSE for details] * \file multi_proposal.cu * \brief MultiProposal Operator * \author Shaoqing Ren, Xizhou Zhu, Jian Guo */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "../operator_common.h" #include "../mshadow_op.h" #include "./multi_proposal-inl.h" #define DIVUP(m, n) ((m) / (n) + ((m) % (n) > 0)) #define FRCNN_CUDA_CHECK(condition) \ /* Code block avoids redefinition of cudaError_t error */ \ do { \ cudaError_t error = condition; \ CHECK_EQ(error, cudaSuccess) << " " << cudaGetErrorString(error); \ } while (0) namespace mshadow { namespace cuda { namespace multi_proposal { // scores are (b, 2 * anchor, h, w) // workspace_proposals are (b, h * w * anchor, 5) // w defines "x" and h defines "y" // count should be total anchors numbers, h * w * anchors template __global__ void ProposalGridKernel(const int count, const int num_anchors, const int height, const int width, const int feature_stride, const Dtype* scores, Dtype* workspace_proposals) { for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < count; index += blockDim.x * gridDim.x) { int a = index % num_anchors; int w = (index / num_anchors) % width; int h = (index / num_anchors / width) % height; int b = index / num_anchors / width / height; workspace_proposals[index * 5 + 0] = workspace_proposals[a * 5 + 0] + w * feature_stride; workspace_proposals[index * 5 + 1] = workspace_proposals[a * 5 + 1] + h * feature_stride; workspace_proposals[index * 5 + 2] = workspace_proposals[a * 5 + 2] + w * feature_stride; workspace_proposals[index * 5 + 3] = workspace_proposals[a * 5 + 3] + h * feature_stride; workspace_proposals[index * 5 + 4] = scores[((b * (2 * num_anchors) + a + num_anchors) * height + h) * width + w]; //workspace_proposals[index * 5 + 4] = scores[(a * height + h) * width + w]; } } // boxes are (b, h * w * anchor, 5) // deltas are (b, 4 * anchor, h, w) // out_pred_boxes are (b, h * w * anchor, 5) // count should be total anchors numbers, b * h * w * anchors // in-place write: boxes and out_pred_boxes are the same location template __global__ void BBoxPredKernel(const int count, const int num_anchors, const int feat_height, const int feat_width, const int feature_stride, const Dtype* im_infos, const Dtype* boxes, const Dtype* deltas, Dtype* out_pred_boxes) { for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < count; index += blockDim.x * gridDim.x) { int a = index % num_anchors; int w = (index / num_anchors) % feat_width; int h = (index / num_anchors / feat_width) % feat_height; int b = index / num_anchors / feat_width / feat_height; float im_height = im_infos[b * 3]; float im_width = im_infos[b * 3 + 1]; int real_height = static_cast(im_height / feature_stride); int real_width = static_cast(im_width / feature_stride); float width = boxes[index * 5 + 2] - boxes[index * 5 + 0] + 1.0f; float height = boxes[index * 5 + 3] - boxes[index * 5 + 1] + 1.0f; float ctr_x = boxes[index * 5 + 0] + 0.5f * (width - 1.0f); float ctr_y = boxes[index * 5 + 1] + 0.5f * (height - 1.0f); int ba = (b * num_anchors + a); float dx = deltas[((ba * 4) * feat_height + h) * feat_width + w]; float dy = deltas[((ba * 4 + 1) * feat_height + h) * feat_width + w]; float dw = deltas[((ba * 4 + 2) * feat_height + h) * feat_width + w]; float dh = deltas[((ba * 4 + 3) * feat_height + h) * feat_width + w]; float pred_ctr_x = dx * width + ctr_x; float pred_ctr_y = dy * height + ctr_y; float pred_w = exp(dw) * width; float pred_h = exp(dh) * height; float pred_x1 = pred_ctr_x - 0.5f * (pred_w - 1.0f); float pred_y1 = pred_ctr_y - 0.5f * (pred_h - 1.0f); float pred_x2 = pred_ctr_x + 0.5f * (pred_w - 1.0f); float pred_y2 = pred_ctr_y + 0.5f * (pred_h - 1.0f); pred_x1 = max(min(pred_x1, im_width - 1.0f), 0.0f); pred_y1 = max(min(pred_y1, im_height - 1.0f), 0.0f); pred_x2 = max(min(pred_x2, im_width - 1.0f), 0.0f); pred_y2 = max(min(pred_y2, im_height - 1.0f), 0.0f); out_pred_boxes[index * 5 + 0] = pred_x1; out_pred_boxes[index * 5 + 1] = pred_y1; out_pred_boxes[index * 5 + 2] = pred_x2; out_pred_boxes[index * 5 + 3] = pred_y2; if (h >= real_height || w >= real_width) { out_pred_boxes[index * 5 + 4] = -1.0f; } } } // boxes are (b, h * w * anchor, 5) // deltas are (b, 4 * anchor, h, w) // out_pred_boxes are (b, h * w * anchor, 5) // count should be total anchors numbers, b * h * w * anchors // in-place write: boxes and out_pred_boxes are the same location template __global__ void IoUPredKernel(const int count, const int num_anchors, const int feat_height, const int feat_width, const int feature_stride, const Dtype* im_infos, const Dtype* boxes, const Dtype* deltas, Dtype* out_pred_boxes) { for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < count; index += blockDim.x * gridDim.x) { int a = index % num_anchors; int w = (index / num_anchors) % feat_width; int h = (index / num_anchors / feat_width) % feat_height; int b = index / num_anchors / feat_width / feat_height; float im_height = im_infos[b * 3]; float im_width = im_infos[b * 3 + 1]; int real_height = static_cast(im_height / feature_stride); int real_width = static_cast(im_width / feature_stride); float x1 = boxes[index * 5 + 0]; float y1 = boxes[index * 5 + 1]; float x2 = boxes[index * 5 + 2]; float y2 = boxes[index * 5 + 3]; int ba = (b * num_anchors + a); float dx1 = deltas[((ba * 4) * feat_height + h) * feat_width + w]; float dy1 = deltas[((ba * 4 + 1) * feat_height + h) * feat_width + w]; float dx2 = deltas[((ba * 4 + 2) * feat_height + h) * feat_width + w]; float dy2 = deltas[((ba * 4 + 3) * feat_height + h) * feat_width + w]; float pred_x1 = max(min(x1 + dx1, im_width - 1.0f), 0.0f); float pred_y1 = max(min(y1 + dy1, im_height - 1.0f), 0.0f); float pred_x2 = max(min(x2 + dx2, im_width - 1.0f), 0.0f); float pred_y2 = max(min(y2 + dy2, im_height - 1.0f), 0.0f); out_pred_boxes[index * 5 + 0] = pred_x1; out_pred_boxes[index * 5 + 1] = pred_y1; out_pred_boxes[index * 5 + 2] = pred_x2; out_pred_boxes[index * 5 + 3] = pred_y2; if (h >= real_height || w >= real_width) { out_pred_boxes[index * 5 + 4] = -1.0f; } } } // filter box with stride less than rpn_min_size // filter: set score to zero // dets (b, n, 5) template __global__ void FilterBoxKernel(const int count, const int count_anchors, const float original_min_size, const Dtype* im_infos, Dtype* dets) { for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < count; index += blockDim.x * gridDim.x) { int b = index / count_anchors; float iw = dets[index * 5 + 2] - dets[index * 5 + 0] + 1.0f; float ih = dets[index * 5 + 3] - dets[index * 5 + 1] + 1.0f; float min_size = original_min_size * im_infos[b * 3 + 2]; if (iw < min_size || ih < min_size) { dets[index * 5 + 0] -= min_size / 2; dets[index * 5 + 1] -= min_size / 2; dets[index * 5 + 2] += min_size / 2; dets[index * 5 + 3] += min_size / 2; dets[index * 5 + 4] = -1.0f; } } } // copy score and init order // dets (n, 5); score (n, ); order (n, ) // count should be n (total anchors or proposals) template __global__ void CopyScoreKernel(const int count, const Dtype* dets, Dtype* score, int* order) { for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < count; index += blockDim.x * gridDim.x) { score[index] = dets[index * 5 + 4]; order[index] = index; } } // reorder proposals according to order and keep the top_n proposals // prev_dets (n, 5); order (n, ); dets (n, 5) // count should be output anchor numbers (top_n) template __global__ void ReorderProposalsKernel(const int count, const Dtype* prev_dets, const int* order, Dtype* dets) { for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < count; index += blockDim.x * gridDim.x) { const int order_i = order[index]; for (int j = 0; j < 5; j ++) { dets[index * 5 + j] = prev_dets[order_i * 5 + j]; } } } __device__ inline float devIoU(float const * const a, float const * const b) { float left = max(a[0], b[0]), right = min(a[2], b[2]); float top = max(a[1], b[1]), bottom = min(a[3], b[3]); float width = max(right - left + 1, 0.f), height = max(bottom - top + 1, 0.f); float interS = width * height; float Sa = (a[2] - a[0] + 1) * (a[3] - a[1] + 1); float Sb = (b[2] - b[0] + 1) * (b[3] - b[1] + 1); return interS / (Sa + Sb - interS); } __global__ void nms_kernel(const int n_boxes, const float nms_overlap_thresh, const float *dev_boxes, uint64_t *dev_mask) { const int threadsPerBlock = sizeof(uint64_t) * 8; const int row_start = blockIdx.y; const int col_start = blockIdx.x; // if (row_start > col_start) return; const int row_size = min(n_boxes - row_start * threadsPerBlock, threadsPerBlock); const int col_size = min(n_boxes - col_start * threadsPerBlock, threadsPerBlock); __shared__ float block_boxes[threadsPerBlock * 5]; if (threadIdx.x < col_size) { block_boxes[threadIdx.x * 5 + 0] = dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 0]; block_boxes[threadIdx.x * 5 + 1] = dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 1]; block_boxes[threadIdx.x * 5 + 2] = dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 2]; block_boxes[threadIdx.x * 5 + 3] = dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 3]; block_boxes[threadIdx.x * 5 + 4] = dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 4]; } __syncthreads(); if (threadIdx.x < row_size) { const int cur_box_idx = threadsPerBlock * row_start + threadIdx.x; const float *cur_box = dev_boxes + cur_box_idx * 5; int i = 0; uint64_t t = 0; int start = 0; if (row_start == col_start) { start = threadIdx.x + 1; } for (i = start; i < col_size; i++) { if (devIoU(cur_box, block_boxes + i * 5) > nms_overlap_thresh) { t |= 1ULL << i; } } const int col_blocks = DIVUP(n_boxes, threadsPerBlock); dev_mask[cur_box_idx * col_blocks + col_start] = t; } } void _nms(const mshadow::Tensor& boxes, const float nms_overlap_thresh, int *keep, int *num_out) { const int threadsPerBlock = sizeof(uint64_t) * 8; const int boxes_num = boxes.size(0); const int boxes_dim = boxes.size(1); float* boxes_dev = boxes.dptr_; uint64_t* mask_dev = NULL; const int col_blocks = DIVUP(boxes_num, threadsPerBlock); FRCNN_CUDA_CHECK(cudaMalloc(&mask_dev, boxes_num * col_blocks * sizeof(uint64_t))); dim3 blocks(DIVUP(boxes_num, threadsPerBlock), DIVUP(boxes_num, threadsPerBlock)); dim3 threads(threadsPerBlock); nms_kernel<<>>(boxes_num, nms_overlap_thresh, boxes_dev, mask_dev); FRCNN_CUDA_CHECK(cudaPeekAtLastError()); std::vector mask_host(boxes_num * col_blocks); FRCNN_CUDA_CHECK(cudaMemcpy(&mask_host[0], mask_dev, sizeof(uint64_t) * boxes_num * col_blocks, cudaMemcpyDeviceToHost)); std::vector remv(col_blocks); memset(&remv[0], 0, sizeof(uint64_t) * col_blocks); int num_to_keep = 0; for (int i = 0; i < boxes_num; i++) { int nblock = i / threadsPerBlock; int inblock = i % threadsPerBlock; if (!(remv[nblock] & (1ULL << inblock))) { keep[num_to_keep++] = i; uint64_t *p = &mask_host[0] + i * col_blocks; for (int j = nblock; j < col_blocks; j++) { remv[j] |= p[j]; } } } *num_out = num_to_keep; FRCNN_CUDA_CHECK(cudaFree(mask_dev)); } // copy proposals to output // dets (top_n, 5); keep (top_n, ); out (top_n, ) // count should be top_n (total anchors or proposals) template __global__ void PrepareOutput(const int count, const Dtype* dets, const int* keep, const int out_size, const int image_index, Dtype* out, Dtype* score) { for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < count; index += blockDim.x * gridDim.x) { out[index * 5] = image_index; if (index < out_size) { int keep_i = keep[index]; for (int j = 0; j < 4; ++j) { out[index * 5 + j + 1] = dets[keep_i * 5 + j]; } score[index] = dets[keep_i * 5 + 4]; } else { int keep_i = keep[index % out_size]; for (int j = 0; j < 4; ++j) { out[index * 5 + j + 1] = dets[keep_i * 5 + j]; } score[index] = dets[keep_i * 5 + 4]; } } } } // namespace multi_proposal } // namespace cuda } // namespace mshadow namespace mxnet { namespace op { template class MultiProposalGPUOp : public Operator{ public: explicit MultiProposalGPUOp(MultiProposalParam param) { this->param_ = param; } virtual void Forward(const OpContext &ctx, const std::vector &in_data, const std::vector &req, const std::vector &out_data, const std::vector &aux_states) { using namespace mshadow; using namespace mshadow::expr; using namespace mshadow::cuda; using namespace mshadow::cuda::multi_proposal; CHECK_EQ(in_data.size(), 3); CHECK_EQ(out_data.size(), 2); CHECK_GT(req.size(), 1); CHECK_EQ(req[proposal::kOut], kWriteTo); /*CHECK_EQ(in_data[proposal::kClsProb].shape_[0], 1) << "Sorry, multiple images each device is not implemented.";*/ Stream *s = ctx.get_stream(); Tensor scores = in_data[proposal::kClsProb].get(s); Tensor bbox_deltas = in_data[proposal::kBBoxPred].get(s); Tensor im_info = in_data[proposal::kImInfo].get(s); Tensor out = out_data[proposal::kOut].get(s); Tensor out_score = out_data[proposal::kScore].get(s); int num_images = scores.size(0); int num_anchors = scores.size(1) / 2; int height = scores.size(2); int width = scores.size(3); int count_anchors = num_anchors * height * width; // count of total anchors int count = num_images * count_anchors; // set to -1 for max int rpn_pre_nms_top_n = (param_.rpn_pre_nms_top_n > 0) ? param_.rpn_pre_nms_top_n : count_anchors; rpn_pre_nms_top_n = std::min(rpn_pre_nms_top_n, count_anchors); int rpn_post_nms_top_n = std::min(param_.rpn_post_nms_top_n, rpn_pre_nms_top_n); // Generate first anchors based on base anchor std::vector base_anchor(4); base_anchor[0] = 0.0; base_anchor[1] = 0.0; base_anchor[2] = param_.feature_stride - 1.0; base_anchor[3] = param_.feature_stride - 1.0; CHECK_EQ(num_anchors, param_.ratios.info.size() * param_.scales.info.size()); std::vector anchors; utils::GenerateAnchors(base_anchor, param_.ratios.info, param_.scales.info, &anchors); // Copy generated anchors to GPU float* workspace_proposals_ptr = NULL; FRCNN_CUDA_CHECK(cudaMalloc(&workspace_proposals_ptr, sizeof(float) * num_images * count_anchors * 5)); Tensor workspace_proposals(workspace_proposals_ptr, Shape3(num_images, count_anchors, 5)); FRCNN_CUDA_CHECK(cudaMemcpy(workspace_proposals.dptr_, &anchors[0], sizeof(float) * anchors.size(), cudaMemcpyHostToDevice)); // Copy proposals to a mesh grid dim3 dimGrid((count + kMaxThreadsPerBlock - 1) / kMaxThreadsPerBlock); dim3 dimBlock(kMaxThreadsPerBlock); CheckLaunchParam(dimGrid, dimBlock, "ProposalGrid"); ProposalGridKernel<<>>( count, num_anchors, height, width, param_.feature_stride, scores.dptr_, workspace_proposals.dptr_); FRCNN_CUDA_CHECK(cudaPeekAtLastError()); // Transform anchors and bbox_deltas into bboxes CheckLaunchParam(dimGrid, dimBlock, "BBoxPred"); if (param_.iou_loss) { IoUPredKernel<<>>( count, num_anchors, height, width, param_.feature_stride, im_info.dptr_, workspace_proposals.dptr_, bbox_deltas.dptr_, workspace_proposals.dptr_); } else { BBoxPredKernel<<>>( count, num_anchors, height, width, param_.feature_stride, im_info.dptr_, workspace_proposals.dptr_, bbox_deltas.dptr_, workspace_proposals.dptr_); } FRCNN_CUDA_CHECK(cudaPeekAtLastError()); // filter boxes with less than rpn_min_size CheckLaunchParam(dimGrid, dimBlock, "FilterBox"); FilterBoxKernel<<>>( count, count_anchors, param_.rpn_min_size, im_info.dptr_, workspace_proposals.dptr_); FRCNN_CUDA_CHECK(cudaPeekAtLastError()); dimGrid = dim3((count_anchors + kMaxThreadsPerBlock - 1) / kMaxThreadsPerBlock); dimBlock = dim3(kMaxThreadsPerBlock); // Copy score to a continuous memory float* score_ptr = NULL; FRCNN_CUDA_CHECK(cudaMalloc(&score_ptr, sizeof(float) * count_anchors)); Tensor score(score_ptr, Shape1(count_anchors)); int* order_ptr = NULL; FRCNN_CUDA_CHECK(cudaMalloc(&order_ptr, sizeof(int) * count_anchors)); Tensor order(order_ptr, Shape1(count_anchors)); float* workspace_ordered_proposals_ptr = NULL; FRCNN_CUDA_CHECK(cudaMalloc(&workspace_ordered_proposals_ptr, sizeof(float) * rpn_pre_nms_top_n * 5)); Tensor workspace_ordered_proposals(workspace_ordered_proposals_ptr, Shape2(rpn_pre_nms_top_n, 5)); int* keep; FRCNN_CUDA_CHECK(cudaMalloc(&keep, sizeof(int) * rpn_pre_nms_top_n)); for (int b = 0; b < num_images; b++) { CheckLaunchParam(dimGrid, dimBlock, "CopyScore"); CopyScoreKernel << > >( count_anchors, workspace_proposals.dptr_ + b * count_anchors * 5, score.dptr_, order.dptr_); FRCNN_CUDA_CHECK(cudaPeekAtLastError()); // argsort score, save order thrust::stable_sort_by_key(thrust::device, score.dptr_, score.dptr_ + score.size(0), order.dptr_, thrust::greater()); FRCNN_CUDA_CHECK(cudaPeekAtLastError()); // Reorder proposals according to order dimGrid.x = (rpn_pre_nms_top_n + kMaxThreadsPerBlock - 1) / kMaxThreadsPerBlock; CheckLaunchParam(dimGrid, dimBlock, "ReorderProposals"); ReorderProposalsKernel << > >( rpn_pre_nms_top_n, workspace_proposals.dptr_ + b * count_anchors * 5, order.dptr_, workspace_ordered_proposals.dptr_); FRCNN_CUDA_CHECK(cudaPeekAtLastError()); // perform nms std::vector _keep(workspace_ordered_proposals.size(0)); int out_size = 0; _nms(workspace_ordered_proposals, param_.threshold, &_keep[0], &out_size); // copy nms result to gpu FRCNN_CUDA_CHECK(cudaMemcpy(keep, &_keep[0], sizeof(int) * _keep.size(), cudaMemcpyHostToDevice)); // copy results after nms dimGrid.x = (rpn_post_nms_top_n + kMaxThreadsPerBlock - 1) / kMaxThreadsPerBlock; CheckLaunchParam(dimGrid, dimBlock, "PrepareOutput"); PrepareOutput << > >( rpn_post_nms_top_n, workspace_ordered_proposals.dptr_, keep, out_size, b, out.dptr_ + b * rpn_post_nms_top_n * 5, out_score.dptr_ + b * rpn_post_nms_top_n); FRCNN_CUDA_CHECK(cudaPeekAtLastError()); } // free temporary memory FRCNN_CUDA_CHECK(cudaFree(keep)); FRCNN_CUDA_CHECK(cudaFree(workspace_ordered_proposals_ptr)); FRCNN_CUDA_CHECK(cudaFree(workspace_proposals_ptr)); FRCNN_CUDA_CHECK(cudaFree(score_ptr)); FRCNN_CUDA_CHECK(cudaFree(order_ptr)); } virtual void Backward(const OpContext &ctx, const std::vector &out_grad, const std::vector &in_data, const std::vector &out_data, const std::vector &req, const std::vector &in_grad, const std::vector &aux_states) { using namespace mshadow; using namespace mshadow::expr; CHECK_EQ(in_grad.size(), 3); Stream *s = ctx.get_stream(); Tensor gscores = in_grad[proposal::kClsProb].get(s); Tensor gbbox = in_grad[proposal::kBBoxPred].get(s); Tensor ginfo = in_grad[proposal::kImInfo].get(s); // can not assume the grad would be zero Assign(gscores, req[proposal::kClsProb], 0); Assign(gbbox, req[proposal::kBBoxPred], 0); Assign(ginfo, req[proposal::kImInfo], 0); } private: MultiProposalParam param_; }; // class MultiProposalGPUOp template<> Operator* CreateOp(MultiProposalParam param) { return new MultiProposalGPUOp(param); } } // namespace op } // namespace mxnet ================================================ FILE: rfcn/operator_cxx/psroi_pooling-inl.h ================================================ /*! * Copyright (c) 2017 by Contributors * Copyright (c) 2017 Microsoft * Licensed under The MIT License [see LICENSE for details] * \file psroi_pooling-inl.h * \brief psroi pooling operator and symbol * \author Yi Li, Tairui Chen, Guodong Zhang, Jifeng Dai */ #ifndef MXNET_OPERATOR_PSROI_POOLING_INL_H_ #define MXNET_OPERATOR_PSROI_POOLING_INL_H_ #include #include #include #include #include #include #include #include "../mshadow_op.h" #include "../operator_common.h" namespace mxnet { namespace op { // Declare enumeration of input order to make code more intuitive. // These enums are only visible within this header namespace psroipool { enum PSROIPoolingOpInputs {kData, kBox}; enum PSROIPoolingOpOutputs {kOut, kMappingChannel}; } // psroipool struct PSROIPoolingParam : public dmlc::Parameter { // TShape pooled_size; float spatial_scale; int output_dim; int pooled_size; int group_size; DMLC_DECLARE_PARAMETER(PSROIPoolingParam) { DMLC_DECLARE_FIELD(spatial_scale).set_range(0.0, 1.0) .describe("Ratio of input feature map height (or w) to raw image height (or w). " "Equals the reciprocal of total stride in convolutional layers"); DMLC_DECLARE_FIELD(output_dim).describe("fix output dim"); DMLC_DECLARE_FIELD(pooled_size).describe("fix pooled size"); DMLC_DECLARE_FIELD(group_size).set_default(0).describe("fix group size"); } }; template class PSROIPoolingOp : public Operator { public: explicit PSROIPoolingOp(PSROIPoolingParam p) { this->param_ = p; } virtual void Forward(const OpContext &ctx, const std::vector &in_data, const std::vector &req, const std::vector &out_data, const std::vector &aux_args) { using namespace mshadow; size_t expected = 2; CHECK_EQ(in_data.size(), expected); CHECK_EQ(out_data.size(), expected); CHECK_EQ(out_data[psroipool::kOut].shape_[0], in_data[psroipool::kBox].shape_[0]); CHECK_EQ(out_data[psroipool::kMappingChannel].shape_[0], in_data[psroipool::kBox].shape_[0]); Stream *s = ctx.get_stream(); Tensor data = in_data[psroipool::kData].get(s); Tensor bbox = in_data[psroipool::kBox].get(s); Tensor out = out_data[psroipool::kOut].get(s); Tensor mapping_channel = out_data[psroipool::kMappingChannel].get(s); CHECK_EQ(data.CheckContiguous(), true); CHECK_EQ(bbox.CheckContiguous(), true); CHECK_EQ(out.CheckContiguous(), true); CHECK_EQ(mapping_channel.CheckContiguous(), true); out = -FLT_MAX; mapping_channel = -1.0f; PSROIPoolForward(out, data, bbox, mapping_channel, param_.spatial_scale, param_.output_dim, param_.group_size); } virtual void Backward(const OpContext &ctx, const std::vector &out_grad, const std::vector &in_data, const std::vector &out_data, const std::vector &req, const std::vector &in_grad, const std::vector &aux_args) { using namespace mshadow; size_t expected = 2; CHECK_EQ(in_data.size(), expected); CHECK_EQ(out_data.size(), expected); CHECK_EQ(out_grad[psroipool::kOut].shape_[0], in_data[psroipool::kBox].shape_[0]); CHECK_EQ(out_data[psroipool::kMappingChannel].shape_[0], in_data[psroipool::kBox].shape_[0]); CHECK_NE(req[psroipool::kData], kWriteInplace) << "ROIPooling: Backward doesn't support kWriteInplace."; CHECK_NE(req[psroipool::kBox], kWriteInplace) << "ROIPooling: Backward doesn't support kWriteInplace."; Stream *s = ctx.get_stream(); Tensor grad_out = out_grad[psroipool::kOut].get(s); Tensor bbox = in_data[psroipool::kBox].get(s); Tensor mapping_channel = out_data[psroipool::kMappingChannel].get(s); Tensor grad_in = in_grad[psroipool::kData].get(s); Tensor grad_roi = in_grad[psroipool::kBox].get(s); CHECK_EQ(grad_out.CheckContiguous(), true); CHECK_EQ(bbox.CheckContiguous(), true); CHECK_EQ(mapping_channel.CheckContiguous(), true); CHECK_EQ(grad_in.CheckContiguous(), true); if (kAddTo == req[psroipool::kData] || kWriteTo == req[psroipool::kData]) { if (kWriteTo == req[psroipool::kData]) { grad_in = 0.0f; } PSROIPoolBackwardAcc(grad_in, grad_out, bbox, mapping_channel, param_.spatial_scale, param_.output_dim); } if (kWriteTo == req[psroipool::kBox]) { grad_roi = 0.0f; } } private: PSROIPoolingParam param_; }; // class PSROIPoolingOp // Decalre Factory function, used for dispatch specialization template Operator* CreateOp(PSROIPoolingParam param, int dtype); #if DMLC_USE_CXX11 class PSROIPoolingProp : public OperatorProperty { public: std::vector ListArguments() const override { return {"data", "rois"}; } std::vector ListOutputs() const override { return {"output", "maxidx"}; } int NumOutputs() const override { return 2; } int NumVisibleOutputs() const override { return 1; } void Init(const std::vector >& kwargs) override { param_.Init(kwargs); if (param_.group_size == 0) { param_.group_size = param_.pooled_size; } } std::map GetParams() const override { return param_.__DICT__(); } bool InferShape(std::vector *in_shape, std::vector *out_shape, std::vector *aux_shape) const override { using namespace mshadow; CHECK_EQ(in_shape->size(), 2) << "Input:[data, rois]"; // data: [batch_size, c, h, w] TShape dshape = in_shape->at(psroipool::kData); CHECK_EQ(dshape.ndim(), 4) << "data should be a 4D tensor"; // bbox: [num_rois, 5] TShape bshape = in_shape->at(psroipool::kBox); CHECK_EQ(bshape.ndim(), 2) << "bbox should be a 2D tensor of shape [batch, 5]"; CHECK_EQ(bshape[1], 5) << "bbox should be a 2D tensor of shape [batch, 5]"; // out: [num_rois, c, pooled_h, pooled_w] // mapping_channel: [num_rois, c, pooled_h, pooled_w] out_shape->clear(); out_shape->push_back( Shape4(bshape[0], param_.output_dim, param_.pooled_size, param_.pooled_size)); out_shape->push_back( Shape4(bshape[0], param_.output_dim, param_.pooled_size, param_.pooled_size)); return true; } bool InferType(std::vector *in_type, std::vector *out_type, std::vector *aux_type) const override { CHECK_EQ(in_type->size(), 2); int dtype = (*in_type)[0]; CHECK_EQ(dtype, (*in_type)[1]); CHECK_NE(dtype, -1) << "Input must have specified type"; out_type->clear(); out_type->push_back(dtype); out_type->push_back(dtype); return true; } OperatorProperty* Copy() const override { PSROIPoolingProp* psroi_pooling_sym = new PSROIPoolingProp(); psroi_pooling_sym->param_ = this->param_; return psroi_pooling_sym; } std::string TypeString() const override { return "_contrib_PSROIPooling"; } // decalre dependency and inplace optimization options std::vector DeclareBackwardDependency( const std::vector &out_grad, const std::vector &in_data, const std::vector &out_data) const override { return {out_grad[psroipool::kOut], in_data[psroipool::kBox], out_data[psroipool::kMappingChannel]}; } Operator* CreateOperator(Context ctx) const override { LOG(FATAL) << "Not Implemented."; return NULL; } Operator* CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const override; private: PSROIPoolingParam param_; }; // class PSROIPoolingProp #endif } // namespace op } // namespace mxnet #endif // MXNET_OPERATOR_PSROI_POOLING_INL_H_ ================================================ FILE: rfcn/operator_cxx/psroi_pooling.cc ================================================ /*! * Copyright (c) 2017 by Contributors * Copyright (c) 2017 Microsoft * Licensed under The MIT License [see LICENSE for details] * \file psroi_pooling.cc * \brief psroi pooling operator * \author Yi Li, Tairui Chen, Guodong Zhang, Jifeng Dai */ #include "./psroi_pooling-inl.h" #include #include #include #include #include using std::max; using std::min; using std::floor; using std::ceil; namespace mshadow { template inline void PSROIPoolForward(const Tensor &out, const Tensor &data, const Tensor &bbox, const Tensor &mapping_channel, const float spatial_scale_, const int output_dim_, const int group_size_) { // NOT_IMPLEMENTED; return; } template inline void PSROIPoolBackwardAcc(const Tensor &in_grad, const Tensor &out_grad, const Tensor &bbox, const Tensor &mapping_channel, const float spatial_scale_, const int output_dim_) { // NOT_IMPLEMENTED; return; } } // namespace mshadow namespace mxnet { namespace op { template<> Operator *CreateOp(PSROIPoolingParam param, int dtype) { Operator* op = NULL; MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { op = new PSROIPoolingOp(param); }); return op; } Operator *PSROIPoolingProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { std::vector out_shape, aux_shape; std::vector out_type, aux_type; CHECK(InferType(in_type, &out_type, &aux_type)); CHECK(InferShape(in_shape, &out_shape, &aux_shape)); DO_BIND_DISPATCH(CreateOp, param_, in_type->at(0)); } DMLC_REGISTER_PARAMETER(PSROIPoolingParam); MXNET_REGISTER_OP_PROPERTY(_contrib_PSROIPooling, PSROIPoolingProp) .describe("Performs region-of-interest pooling on inputs. Resize bounding box coordinates by " "spatial_scale and crop input feature maps accordingly. The cropped feature maps are pooled " "by max pooling to a fixed size output indicated by pooled_size. batch_size will change to " "the number of region bounding boxes after PSROIPooling") .add_argument("data", "Symbol", "Input data to the pooling operator, a 4D Feature maps") .add_argument("rois", "Symbol", "Bounding box coordinates, a 2D array of " "[[batch_index, x1, y1, x2, y2]]. (x1, y1) and (x2, y2) are top left and down right corners " "of designated region of interest. batch_index indicates the index of corresponding image " "in the input data") .add_arguments(PSROIPoolingParam::__FIELDS__()); } // namespace op } // namespace mxnet ================================================ FILE: rfcn/operator_cxx/psroi_pooling.cu ================================================ /*! * Copyright (c) 2017 by Contributors * Copyright (c) 2017 Microsoft * Licensed under The MIT License [see LICENSE for details] * \file psroi_pooling.cu * \brief psroi pooling operator * \author Yi Li, Tairui Chen, Guodong Zhang, Jifeng Dai */ #include "./psroi_pooling-inl.h" #include #include #include #include #include "../../common/cuda_utils.h" #include "../mxnet_op.h" #define PSROIPOOLING_CUDA_CHECK(condition) \ /* Code block avoids redefinition of cudaError_t error */ \ do { \ cudaError_t error = condition; \ CHECK_EQ(error, cudaSuccess) << " " << cudaGetErrorString(error); \ } while (0) #define CUDA_KERNEL_LOOP(i, n) \ for (int i = blockIdx.x * blockDim.x + threadIdx.x; \ i < (n); \ i += blockDim.x * gridDim.x) namespace mshadow { namespace cuda { template __global__ void PSROIPoolForwardKernel( const int count, const DType* bottom_data, const DType spatial_scale, const int channels, const int height, const int width, const int pooled_height, const int pooled_width, const DType* bottom_rois, const int output_dim, const int group_size, DType* top_data, DType* mapping_channel) { CUDA_KERNEL_LOOP(index, count) { // The output is in order (n, ctop, ph, pw) int pw = index % pooled_width; int ph = (index / pooled_width) % pooled_height; int ctop = (index / pooled_width / pooled_height) % output_dim; int n = index / pooled_width / pooled_height / output_dim; // [start, end) interval for spatial sampling bottom_rois += n * 5; int roi_batch_ind = bottom_rois[0]; DType roi_start_w = static_cast(round(bottom_rois[1])) * spatial_scale; DType roi_start_h = static_cast(round(bottom_rois[2])) * spatial_scale; DType roi_end_w = static_cast(round(bottom_rois[3]) + 1.) * spatial_scale; DType roi_end_h = static_cast(round(bottom_rois[4]) + 1.) * spatial_scale; // Force too small ROIs to be 1x1 DType roi_width = max(roi_end_w - roi_start_w, 0.1); //avoid 0 DType roi_height = max(roi_end_h - roi_start_h, 0.1); // Compute w and h at bottom DType bin_size_h = roi_height / static_cast(pooled_height); DType bin_size_w = roi_width / static_cast(pooled_width); int hstart = floor(static_cast(ph) * bin_size_h + roi_start_h); int wstart = floor(static_cast(pw)* bin_size_w + roi_start_w); int hend = ceil(static_cast(ph + 1) * bin_size_h + roi_start_h); int wend = ceil(static_cast(pw + 1) * bin_size_w + roi_start_w); // Add roi offsets and clip to input boundaries hstart = min(max(hstart, 0), height); hend = min(max(hend, 0), height); wstart = min(max(wstart, 0),width); wend = min(max(wend, 0), width); bool is_empty = (hend <= hstart) || (wend <= wstart); int gw = floor(static_cast(pw)* group_size / pooled_width); int gh = floor(static_cast(ph)* group_size / pooled_height); gw = min(max(gw, 0), group_size - 1); gh = min(max(gh, 0), group_size - 1); int c = (ctop*group_size + gh)*group_size + gw; bottom_data += (roi_batch_ind * channels + c) * height * width; DType out_sum = 0; for (int h = hstart; h < hend; ++h){ for (int w = wstart; w < wend; ++w){ int bottom_index = h*width + w; out_sum += bottom_data[bottom_index]; } } DType bin_area = (hend - hstart)*(wend - wstart); top_data[index] = is_empty? (DType)0. : out_sum/bin_area; mapping_channel[index] = c; } } template inline void PSROIPoolForward(const Tensor &out, const Tensor &data, const Tensor &bbox, const Tensor &mapping_channel, const float spatial_scale, const int output_dim_, const int group_size_) { // LOG(INFO) << "PSROIPoolForward"; const DType *bottom_data = data.dptr_; const DType *bottom_rois = bbox.dptr_; DType *top_data = out.dptr_; DType *mapping_channel_ptr = mapping_channel.dptr_; const int count = out.shape_.Size(); const int channels = data.size(1); const int height = data.size(2); const int width = data.size(3); const int pooled_height = out.size(2); const int pooled_width = out.size(3); cudaStream_t stream = Stream::GetStream(out.stream_); PSROIPoolForwardKernel << > >( count, bottom_data, spatial_scale, channels, height, width, pooled_height, pooled_width, bottom_rois, output_dim_, group_size_, top_data, mapping_channel_ptr); PSROIPOOLING_CUDA_CHECK(cudaPeekAtLastError()); } template __global__ void PSROIPoolBackwardAccKernel( const int count, const DType* top_diff, const DType* mapping_channel, const int num_rois, const DType spatial_scale, const int channels, const int height, const int width, const int pooled_height, const int pooled_width, const int output_dim, DType* bottom_diff, const DType* bottom_rois) { CUDA_KERNEL_LOOP(index, count) { // The output is in order (n, ctop, ph, pw) int pw = index % pooled_width; int ph = (index / pooled_width) % pooled_height; int n = index / pooled_width / pooled_height / output_dim; // [start, end) interval for spatial sampling bottom_rois += n * 5; int roi_batch_ind = bottom_rois[0]; DType roi_start_w = static_cast(round(bottom_rois[1])) * spatial_scale; DType roi_start_h = static_cast(round(bottom_rois[2])) * spatial_scale; DType roi_end_w = static_cast(round(bottom_rois[3]) + 1.) * spatial_scale; DType roi_end_h = static_cast(round(bottom_rois[4]) + 1.) * spatial_scale; // Force too small ROIs to be 1x1 DType roi_width = max(roi_end_w - roi_start_w, 0.1); //avoid 0 DType roi_height = max(roi_end_h - roi_start_h, 0.1); // Compute w and h at bottom DType bin_size_h = roi_height / static_cast(pooled_height); DType bin_size_w = roi_width / static_cast(pooled_width); int hstart = floor(static_cast(ph)* bin_size_h + roi_start_h); int wstart = floor(static_cast(pw)* bin_size_w + roi_start_w); int hend = ceil(static_cast(ph + 1) * bin_size_h + roi_start_h); int wend = ceil(static_cast(pw + 1) * bin_size_w + roi_start_w); // Add roi offsets and clip to input boundaries hstart = min(max(hstart, 0), height); hend = min(max(hend, 0), height); wstart = min(max(wstart, 0), width); wend = min(max(wend, 0), width); bool is_empty = (hend <= hstart) || (wend <= wstart); // Compute c at bottom int c = mapping_channel[index]; DType* offset_bottom_diff = bottom_diff + (roi_batch_ind * channels + c) * height * width; DType bin_area = (hend - hstart)*(wend - wstart); DType diff_val = is_empty ? (DType)0. : top_diff[index] / bin_area; for (int h = hstart; h < hend; ++h){ for (int w = wstart; w < wend; ++w){ int bottom_index = h*width + w; // mxnet_gpu_atomic_add(diff_val, offset_bottom_diff + bottom_index); atomicAdd(offset_bottom_diff + bottom_index, diff_val); } } } } template inline void PSROIPoolBackwardAcc(const Tensor &in_grad, const Tensor &out_grad, const Tensor &bbox, const Tensor &mapping_channel, const float spatial_scale, const int output_dim_) { // LOG(INFO) << "PSROIPoolBackward"; const DType *top_diff = out_grad.dptr_; const DType *bottom_rois = bbox.dptr_; DType *bottom_diff = in_grad.dptr_; DType *mapping_channel_ptr = mapping_channel.dptr_; const int count = out_grad.shape_.Size(); const int num_rois = bbox.size(0); const int channels = in_grad.size(1); const int height = in_grad.size(2); const int width = in_grad.size(3); const int pooled_height = out_grad.size(2); const int pooled_width = out_grad.size(3); cudaStream_t stream = Stream::GetStream(in_grad.stream_); PSROIPoolBackwardAccKernel << > >( count, top_diff, mapping_channel_ptr, num_rois, spatial_scale, channels, height, width, pooled_height, pooled_width, output_dim_, bottom_diff, bottom_rois); PSROIPOOLING_CUDA_CHECK(cudaPeekAtLastError()); } } // namespace cuda template inline void PSROIPoolForward(const Tensor &out, const Tensor &data, const Tensor &bbox, const Tensor &mapping_channel, const float spatial_scale, const int output_dim_, const int group_size_) { cuda::PSROIPoolForward(out, data, bbox, mapping_channel, spatial_scale, output_dim_, group_size_); } template inline void PSROIPoolBackwardAcc(const Tensor &in_grad, const Tensor &out_grad, const Tensor &bbox, const Tensor &mapping_channel, const float spatial_scale, const int output_dim_) { cuda::PSROIPoolBackwardAcc(in_grad, out_grad, bbox, mapping_channel, spatial_scale, output_dim_); } } // namespace mshadow namespace mxnet { namespace op { template<> Operator* CreateOp(PSROIPoolingParam param, int dtype) { Operator* op = NULL; MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { op = new PSROIPoolingOp(param); }); return op; } } // namespace op } // namespace mxnet ================================================ FILE: rfcn/operator_py/__init__.py ================================================ ================================================ FILE: rfcn/operator_py/box_annotator_ohem.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong # -------------------------------------------------------- """ Proposal Target Operator selects foreground and background roi and assigns label, bbox_transform to them. """ import mxnet as mx import numpy as np from distutils.util import strtobool class BoxAnnotatorOHEMOperator(mx.operator.CustomOp): def __init__(self, num_classes, num_reg_classes, roi_per_img): super(BoxAnnotatorOHEMOperator, self).__init__() self._num_classes = num_classes self._num_reg_classes = num_reg_classes self._roi_per_img = roi_per_img def forward(self, is_train, req, in_data, out_data, aux): cls_score = in_data[0] bbox_pred = in_data[1] labels = in_data[2].asnumpy() bbox_targets = in_data[3] bbox_weights = in_data[4] per_roi_loss_cls = mx.nd.SoftmaxActivation(cls_score) + 1e-14 per_roi_loss_cls = per_roi_loss_cls.asnumpy() per_roi_loss_cls = per_roi_loss_cls[np.arange(per_roi_loss_cls.shape[0], dtype='int'), labels.astype('int')] per_roi_loss_cls = -1 * np.log(per_roi_loss_cls) per_roi_loss_cls = np.reshape(per_roi_loss_cls, newshape=(-1,)) per_roi_loss_bbox = bbox_weights * mx.nd.smooth_l1((bbox_pred - bbox_targets), scalar=1.0) per_roi_loss_bbox = mx.nd.sum(per_roi_loss_bbox, axis=1).asnumpy() top_k_per_roi_loss = np.argsort(per_roi_loss_cls + per_roi_loss_bbox) labels_ohem = labels labels_ohem[top_k_per_roi_loss[::-1][self._roi_per_img:]] = -1 bbox_weights_ohem = bbox_weights.asnumpy() bbox_weights_ohem[top_k_per_roi_loss[::-1][self._roi_per_img:]] = 0 labels_ohem = mx.nd.array(labels_ohem) bbox_weights_ohem = mx.nd.array(bbox_weights_ohem) for ind, val in enumerate([labels_ohem, bbox_weights_ohem]): self.assign(out_data[ind], req[ind], val) def backward(self, req, out_grad, in_data, out_data, in_grad, aux): for i in range(len(in_grad)): self.assign(in_grad[i], req[i], 0) @mx.operator.register('BoxAnnotatorOHEM') class BoxAnnotatorOHEMProp(mx.operator.CustomOpProp): def __init__(self, num_classes, num_reg_classes, roi_per_img): super(BoxAnnotatorOHEMProp, self).__init__(need_top_grad=False) self._num_classes = int(num_classes) self._num_reg_classes = int(num_reg_classes) self._roi_per_img = int(roi_per_img) def list_arguments(self): return ['cls_score', 'bbox_pred', 'labels', 'bbox_targets', 'bbox_weights'] def list_outputs(self): return ['labels_ohem', 'bbox_weights_ohem'] def infer_shape(self, in_shape): labels_shape = in_shape[2] bbox_weights_shape = in_shape[4] return in_shape, \ [labels_shape, bbox_weights_shape] def create_operator(self, ctx, shapes, dtypes): return BoxAnnotatorOHEMOperator(self._num_classes, self._num_reg_classes, self._roi_per_img) def declare_backward_dependency(self, out_grad, in_data, out_data): return [] ================================================ FILE: rfcn/operator_py/proposal.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- """ Proposal Operator transform anchor coordinates into ROI coordinates with prediction results on classification probability and bounding box prediction results, and image size and scale information. """ import mxnet as mx import numpy as np import numpy.random as npr from distutils.util import strtobool from bbox.bbox_transform import bbox_pred, clip_boxes from rpn.generate_anchor import generate_anchors from nms.nms import py_nms_wrapper, cpu_nms_wrapper, gpu_nms_wrapper DEBUG = False class ProposalOperator(mx.operator.CustomOp): def __init__(self, feat_stride, scales, ratios, output_score, rpn_pre_nms_top_n, rpn_post_nms_top_n, threshold, rpn_min_size): super(ProposalOperator, self).__init__() self._feat_stride = feat_stride self._scales = np.fromstring(scales[1:-1], dtype=float, sep=',') self._ratios = np.fromstring(ratios[1:-1], dtype=float, sep=',') self._anchors = generate_anchors(base_size=self._feat_stride, scales=self._scales, ratios=self._ratios) self._num_anchors = self._anchors.shape[0] self._output_score = output_score self._rpn_pre_nms_top_n = rpn_pre_nms_top_n self._rpn_post_nms_top_n = rpn_post_nms_top_n self._threshold = threshold self._rpn_min_size = rpn_min_size if DEBUG: print 'feat_stride: {}'.format(self._feat_stride) print 'anchors:' print self._anchors def forward(self, is_train, req, in_data, out_data, aux): nms = gpu_nms_wrapper(self._threshold, in_data[0].context.device_id) batch_size = in_data[0].shape[0] if batch_size > 1: raise ValueError("Sorry, multiple images each device is not implemented") # for each (H, W) location i # generate A anchor boxes centered on cell i # apply predicted bbox deltas at cell i to each of the A anchors # clip predicted boxes to image # remove predicted boxes with either height or width < threshold # sort all (proposal, score) pairs by score from highest to lowest # take top pre_nms_topN proposals before NMS # apply NMS with threshold 0.7 to remaining proposals # take after_nms_topN proposals after NMS # return the top proposals (-> RoIs top, scores top) pre_nms_topN = self._rpn_pre_nms_top_n post_nms_topN = self._rpn_post_nms_top_n min_size = self._rpn_min_size # the first set of anchors are background probabilities # keep the second part scores = in_data[0].asnumpy()[:, self._num_anchors:, :, :] bbox_deltas = in_data[1].asnumpy() im_info = in_data[2].asnumpy()[0, :] if DEBUG: print 'im_size: ({}, {})'.format(im_info[0], im_info[1]) print 'scale: {}'.format(im_info[2]) # 1. Generate proposals from bbox_deltas and shifted anchors # use real image size instead of padded feature map sizes height, width = int(im_info[0] / self._feat_stride), int(im_info[1] / self._feat_stride) if DEBUG: print 'score map size: {}'.format(scores.shape) print "resudial: {}".format((scores.shape[2] - height, scores.shape[3] - width)) # Enumerate all shifts shift_x = np.arange(0, width) * self._feat_stride shift_y = np.arange(0, height) * self._feat_stride shift_x, shift_y = np.meshgrid(shift_x, shift_y) shifts = np.vstack((shift_x.ravel(), shift_y.ravel(), shift_x.ravel(), shift_y.ravel())).transpose() # Enumerate all shifted anchors: # # add A anchors (1, A, 4) to # cell K shifts (K, 1, 4) to get # shift anchors (K, A, 4) # reshape to (K*A, 4) shifted anchors A = self._num_anchors K = shifts.shape[0] anchors = self._anchors.reshape((1, A, 4)) + shifts.reshape((1, K, 4)).transpose((1, 0, 2)) anchors = anchors.reshape((K * A, 4)) # Transpose and reshape predicted bbox transformations to get them # into the same order as the anchors: # # bbox deltas will be (1, 4 * A, H, W) format # transpose to (1, H, W, 4 * A) # reshape to (1 * H * W * A, 4) where rows are ordered by (h, w, a) # in slowest to fastest order bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4)) # Same story for the scores: # # scores are (1, A, H, W) format # transpose to (1, H, W, A) # reshape to (1 * H * W * A, 1) where rows are ordered by (h, w, a) scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) # Convert anchors into proposals via bbox transformations proposals = bbox_pred(anchors, bbox_deltas) # 2. clip predicted boxes to image proposals = clip_boxes(proposals, im_info[:2]) # 3. remove predicted boxes with either height or width < threshold # (NOTE: convert min_size to input image scale stored in im_info[2]) keep = self._filter_boxes(proposals, min_size * im_info[2]) proposals = proposals[keep, :] scores = scores[keep] # 4. sort all (proposal, score) pairs by score from highest to lowest # 5. take top pre_nms_topN (e.g. 6000) order = scores.ravel().argsort()[::-1] if pre_nms_topN > 0: order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] # 6. apply nms (e.g. threshold = 0.7) # 7. take after_nms_topN (e.g. 300) # 8. return the top proposals (-> RoIs top) det = np.hstack((proposals, scores)).astype(np.float32) keep = nms(det) if post_nms_topN > 0: keep = keep[:post_nms_topN] # pad to ensure output size remains unchanged if len(keep) < post_nms_topN: pad = npr.choice(keep, size=post_nms_topN - len(keep)) keep = np.hstack((keep, pad)) proposals = proposals[keep, :] scores = scores[keep] # Output rois array # Our RPN implementation only supports a single input image, so all # batch inds are 0 batch_inds = np.zeros((proposals.shape[0], 1), dtype=np.float32) blob = np.hstack((batch_inds, proposals.astype(np.float32, copy=False))) self.assign(out_data[0], req[0], blob) if self._output_score: self.assign(out_data[1], req[1], scores.astype(np.float32, copy=False)) def backward(self, req, out_grad, in_data, out_data, in_grad, aux): self.assign(in_grad[0], req[0], 0) self.assign(in_grad[1], req[1], 0) self.assign(in_grad[2], req[2], 0) @staticmethod def _filter_boxes(boxes, min_size): """ Remove all boxes with any side smaller than min_size """ ws = boxes[:, 2] - boxes[:, 0] + 1 hs = boxes[:, 3] - boxes[:, 1] + 1 keep = np.where((ws >= min_size) & (hs >= min_size))[0] return keep @staticmethod def _clip_pad(tensor, pad_shape): """ Clip boxes of the pad area. :param tensor: [n, c, H, W] :param pad_shape: [h, w] :return: [n, c, h, w] """ H, W = tensor.shape[2:] h, w = pad_shape if h < H or w < W: tensor = tensor[:, :, :h, :w].copy() return tensor @mx.operator.register("proposal") class ProposalProp(mx.operator.CustomOpProp): def __init__(self, feat_stride='16', scales='(8, 16, 32)', ratios='(0.5, 1, 2)', output_score='False', rpn_pre_nms_top_n='6000', rpn_post_nms_top_n='300', threshold='0.3', rpn_min_size='16'): super(ProposalProp, self).__init__(need_top_grad=False) self._feat_stride = int(feat_stride) self._scales = scales self._ratios = ratios self._output_score = strtobool(output_score) self._rpn_pre_nms_top_n = int(rpn_pre_nms_top_n) self._rpn_post_nms_top_n = int(rpn_post_nms_top_n) self._threshold = float(threshold) self._rpn_min_size = int(rpn_min_size) def list_arguments(self): return ['cls_prob', 'bbox_pred', 'im_info'] def list_outputs(self): if self._output_score: return ['output', 'score'] else: return ['output'] def infer_shape(self, in_shape): cls_prob_shape = in_shape[0] bbox_pred_shape = in_shape[1] assert cls_prob_shape[0] == bbox_pred_shape[0], 'ROI number does not equal in cls and reg' batch_size = cls_prob_shape[0] im_info_shape = (batch_size, 3) output_shape = (self._rpn_post_nms_top_n, 5) score_shape = (self._rpn_post_nms_top_n, 1) if self._output_score: return [cls_prob_shape, bbox_pred_shape, im_info_shape], [output_shape, score_shape] else: return [cls_prob_shape, bbox_pred_shape, im_info_shape], [output_shape] def create_operator(self, ctx, shapes, dtypes): return ProposalOperator(self._feat_stride, self._scales, self._ratios, self._output_score, self._rpn_pre_nms_top_n, self._rpn_post_nms_top_n, self._threshold, self._rpn_min_size) def declare_backward_dependency(self, out_grad, in_data, out_data): return [] ================================================ FILE: rfcn/operator_py/proposal_target.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- """ Proposal Target Operator selects foreground and background roi and assigns label, bbox_transform to them. """ import mxnet as mx import numpy as np from distutils.util import strtobool from easydict import EasyDict as edict import cPickle from core.rcnn import sample_rois DEBUG = False class ProposalTargetOperator(mx.operator.CustomOp): def __init__(self, num_classes, batch_images, batch_rois, cfg, fg_fraction): super(ProposalTargetOperator, self).__init__() self._num_classes = num_classes self._batch_images = batch_images self._batch_rois = batch_rois self._cfg = cfg self._fg_fraction = fg_fraction if DEBUG: self._count = 0 self._fg_num = 0 self._bg_num = 0 def forward(self, is_train, req, in_data, out_data, aux): assert self._batch_rois == -1 or self._batch_rois % self._batch_images == 0, \ 'batchimages {} must devide batch_rois {}'.format(self._batch_images, self._batch_rois) all_rois = in_data[0].asnumpy() gt_boxes = in_data[1].asnumpy() if self._batch_rois == -1: rois_per_image = all_rois.shape[0] + gt_boxes.shape[0] fg_rois_per_image = rois_per_image else: rois_per_image = self._batch_rois / self._batch_images fg_rois_per_image = np.round(self._fg_fraction * rois_per_image).astype(int) # Include ground-truth boxes in the set of candidate rois zeros = np.zeros((gt_boxes.shape[0], 1), dtype=gt_boxes.dtype) all_rois = np.vstack((all_rois, np.hstack((zeros, gt_boxes[:, :-1])))) # Sanity check: single batch only assert np.all(all_rois[:, 0] == 0), 'Only single item batches are supported' rois, labels, bbox_targets, bbox_weights = \ sample_rois(all_rois, fg_rois_per_image, rois_per_image, self._num_classes, self._cfg, gt_boxes=gt_boxes) if DEBUG: print "labels=", labels print 'num fg: {}'.format((labels > 0).sum()) print 'num bg: {}'.format((labels == 0).sum()) self._count += 1 self._fg_num += (labels > 0).sum() self._bg_num += (labels == 0).sum() print "self._count=", self._count print 'num fg avg: {}'.format(self._fg_num / self._count) print 'num bg avg: {}'.format(self._bg_num / self._count) print 'ratio: {:.3f}'.format(float(self._fg_num) / float(self._bg_num)) for ind, val in enumerate([rois, labels, bbox_targets, bbox_weights]): self.assign(out_data[ind], req[ind], val) def backward(self, req, out_grad, in_data, out_data, in_grad, aux): self.assign(in_grad[0], req[0], 0) self.assign(in_grad[1], req[1], 0) @mx.operator.register('proposal_target') class ProposalTargetProp(mx.operator.CustomOpProp): def __init__(self, num_classes, batch_images, batch_rois, cfg, fg_fraction='0.25'): super(ProposalTargetProp, self).__init__(need_top_grad=False) self._num_classes = int(num_classes) self._batch_images = int(batch_images) self._batch_rois = int(batch_rois) self._cfg = cPickle.loads(cfg) self._fg_fraction = float(fg_fraction) def list_arguments(self): return ['rois', 'gt_boxes'] def list_outputs(self): return ['rois_output', 'label', 'bbox_target', 'bbox_weight'] def infer_shape(self, in_shape): rpn_rois_shape = in_shape[0] gt_boxes_shape = in_shape[1] rois = rpn_rois_shape[0] + gt_boxes_shape[0] if self._batch_rois == -1 else self._batch_rois output_rois_shape = (rois, 5) label_shape = (rois, ) bbox_target_shape = (rois, self._num_classes * 4) bbox_weight_shape = (rois, self._num_classes * 4) return [rpn_rois_shape, gt_boxes_shape], \ [output_rois_shape, label_shape, bbox_target_shape, bbox_weight_shape] def create_operator(self, ctx, shapes, dtypes): return ProposalTargetOperator(self._num_classes, self._batch_images, self._batch_rois, self._cfg, self._fg_fraction) def declare_backward_dependency(self, out_grad, in_data, out_data): return [] ================================================ FILE: rfcn/operator_py/rpn_inv_normalize.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Xizhou Zhu # -------------------------------------------------------- import mxnet as mx import numpy as np from distutils.util import strtobool class RPNInvNormalizeOperator(mx.operator.CustomOp): def __init__(self, num_anchors, bbox_mean, bbox_std): super(RPNInvNormalizeOperator, self).__init__() self._num_anchors = num_anchors self._bbox_mean = mx.ndarray.Reshape(mx.nd.array(bbox_mean), shape=(1,4,1,1)) self._bbox_std = mx.ndarray.Reshape(mx.nd.array(bbox_std), shape=(1,4,1,1)) def forward(self, is_train, req, in_data, out_data, aux): bbox_pred = in_data[0] tile_shape = (bbox_pred.shape[0], self._num_anchors, bbox_pred.shape[2], bbox_pred.shape[3]) bbox_mean = mx.ndarray.tile(self._bbox_mean.as_in_context(bbox_pred.context), reps=tile_shape) bbox_std = mx.ndarray.tile(self._bbox_std.as_in_context(bbox_pred.context), reps=tile_shape) bbox_pred = bbox_pred * bbox_std + bbox_mean self.assign(out_data[0], req[0], bbox_pred) def backward(self, req, out_grad, in_data, out_data, in_grad, aux): self.assign(in_grad[0], req[0], 0) @mx.operator.register('rpn_inv_normalize') class RPNInvNormalizeProp(mx.operator.CustomOpProp): def __init__(self, num_anchors, bbox_mean='(0.0, 0.0, 0.0, 0.0)', bbox_std='0.1, 0.1, 0.2, 0.2'): super(RPNInvNormalizeProp, self).__init__(need_top_grad=False) self._num_anchors = int(num_anchors) self._bbox_mean = np.fromstring(bbox_mean[1:-1], dtype=float, sep=',') self._bbox_std = np.fromstring(bbox_std[1:-1], dtype=float, sep=',') def list_arguments(self): return ['bbox_pred'] def list_outputs(self): return ['out_bbox_pred'] def infer_shape(self, in_shape): return [in_shape[0]], \ [in_shape[0]] def create_operator(self, ctx, shapes, dtypes): return RPNInvNormalizeOperator(self._num_anchors, self._bbox_mean, self._bbox_std) def declare_backward_dependency(self, out_grad, in_data, out_data): return [] ================================================ FILE: rfcn/symbols/__init__.py ================================================ import resnet_v1_101_rfcn ================================================ FILE: rfcn/symbols/resnet_v1_101_rfcn.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Written by Yuwen Xiong, Xizhou Zhu # -------------------------------------------------------- import cPickle import mxnet as mx from utils.symbol import Symbol from operator_py.proposal import * from operator_py.proposal_target import * from operator_py.box_annotator_ohem import * from operator_py.rpn_inv_normalize import * class resnet_v1_101_rfcn(Symbol): def __init__(self): """ Use __init__ to define parameter network needs """ self.eps = 1e-5 self.use_global_stats = True self.workspace = 512 self.units = (3, 4, 23, 3) # use for 101 self.filter_list = [256, 512, 1024, 2048] def get_resnet_v1(self, data): conv1 = mx.symbol.Convolution(name='conv1', data=data , num_filter=64, pad=(3,3), kernel=(7,7), stride=(2,2), no_bias=True) bn_conv1 = mx.symbol.BatchNorm(name='bn_conv1', data=conv1 , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale_conv1 = bn_conv1 conv1_relu = mx.symbol.Activation(name='conv1_relu', data=scale_conv1 , act_type='relu') pool1 = mx.symbol.Pooling(name='pool1', data=conv1_relu , pad=(1,1), kernel=(3,3), stride=(2,2), pool_type='max') res2a_branch1 = mx.symbol.Convolution(name='res2a_branch1', data=pool1 , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn2a_branch1 = mx.symbol.BatchNorm(name='bn2a_branch1', data=res2a_branch1 , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale2a_branch1 = bn2a_branch1 res2a_branch2a = mx.symbol.Convolution(name='res2a_branch2a', data=pool1 , num_filter=64, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn2a_branch2a = mx.symbol.BatchNorm(name='bn2a_branch2a', data=res2a_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale2a_branch2a = bn2a_branch2a res2a_branch2a_relu = mx.symbol.Activation(name='res2a_branch2a_relu', data=scale2a_branch2a , act_type='relu') res2a_branch2b = mx.symbol.Convolution(name='res2a_branch2b', data=res2a_branch2a_relu , num_filter=64, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn2a_branch2b = mx.symbol.BatchNorm(name='bn2a_branch2b', data=res2a_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale2a_branch2b = bn2a_branch2b res2a_branch2b_relu = mx.symbol.Activation(name='res2a_branch2b_relu', data=scale2a_branch2b , act_type='relu') res2a_branch2c = mx.symbol.Convolution(name='res2a_branch2c', data=res2a_branch2b_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn2a_branch2c = mx.symbol.BatchNorm(name='bn2a_branch2c', data=res2a_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale2a_branch2c = bn2a_branch2c res2a = mx.symbol.broadcast_add(name='res2a', *[scale2a_branch1,scale2a_branch2c] ) res2a_relu = mx.symbol.Activation(name='res2a_relu', data=res2a , act_type='relu') res2b_branch2a = mx.symbol.Convolution(name='res2b_branch2a', data=res2a_relu , num_filter=64, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn2b_branch2a = mx.symbol.BatchNorm(name='bn2b_branch2a', data=res2b_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale2b_branch2a = bn2b_branch2a res2b_branch2a_relu = mx.symbol.Activation(name='res2b_branch2a_relu', data=scale2b_branch2a , act_type='relu') res2b_branch2b = mx.symbol.Convolution(name='res2b_branch2b', data=res2b_branch2a_relu , num_filter=64, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn2b_branch2b = mx.symbol.BatchNorm(name='bn2b_branch2b', data=res2b_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale2b_branch2b = bn2b_branch2b res2b_branch2b_relu = mx.symbol.Activation(name='res2b_branch2b_relu', data=scale2b_branch2b , act_type='relu') res2b_branch2c = mx.symbol.Convolution(name='res2b_branch2c', data=res2b_branch2b_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn2b_branch2c = mx.symbol.BatchNorm(name='bn2b_branch2c', data=res2b_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale2b_branch2c = bn2b_branch2c res2b = mx.symbol.broadcast_add(name='res2b', *[res2a_relu,scale2b_branch2c] ) res2b_relu = mx.symbol.Activation(name='res2b_relu', data=res2b , act_type='relu') res2c_branch2a = mx.symbol.Convolution(name='res2c_branch2a', data=res2b_relu , num_filter=64, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn2c_branch2a = mx.symbol.BatchNorm(name='bn2c_branch2a', data=res2c_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale2c_branch2a = bn2c_branch2a res2c_branch2a_relu = mx.symbol.Activation(name='res2c_branch2a_relu', data=scale2c_branch2a , act_type='relu') res2c_branch2b = mx.symbol.Convolution(name='res2c_branch2b', data=res2c_branch2a_relu , num_filter=64, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn2c_branch2b = mx.symbol.BatchNorm(name='bn2c_branch2b', data=res2c_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale2c_branch2b = bn2c_branch2b res2c_branch2b_relu = mx.symbol.Activation(name='res2c_branch2b_relu', data=scale2c_branch2b , act_type='relu') res2c_branch2c = mx.symbol.Convolution(name='res2c_branch2c', data=res2c_branch2b_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn2c_branch2c = mx.symbol.BatchNorm(name='bn2c_branch2c', data=res2c_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale2c_branch2c = bn2c_branch2c res2c = mx.symbol.broadcast_add(name='res2c', *[res2b_relu,scale2c_branch2c] ) res2c_relu = mx.symbol.Activation(name='res2c_relu', data=res2c , act_type='relu') res3a_branch1 = mx.symbol.Convolution(name='res3a_branch1', data=res2c_relu , num_filter=512, pad=(0,0), kernel=(1,1), stride=(2,2), no_bias=True) bn3a_branch1 = mx.symbol.BatchNorm(name='bn3a_branch1', data=res3a_branch1 , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3a_branch1 = bn3a_branch1 res3a_branch2a = mx.symbol.Convolution(name='res3a_branch2a', data=res2c_relu , num_filter=128, pad=(0,0), kernel=(1,1), stride=(2,2), no_bias=True) bn3a_branch2a = mx.symbol.BatchNorm(name='bn3a_branch2a', data=res3a_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3a_branch2a = bn3a_branch2a res3a_branch2a_relu = mx.symbol.Activation(name='res3a_branch2a_relu', data=scale3a_branch2a , act_type='relu') res3a_branch2b = mx.symbol.Convolution(name='res3a_branch2b', data=res3a_branch2a_relu , num_filter=128, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn3a_branch2b = mx.symbol.BatchNorm(name='bn3a_branch2b', data=res3a_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3a_branch2b = bn3a_branch2b res3a_branch2b_relu = mx.symbol.Activation(name='res3a_branch2b_relu', data=scale3a_branch2b , act_type='relu') res3a_branch2c = mx.symbol.Convolution(name='res3a_branch2c', data=res3a_branch2b_relu , num_filter=512, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn3a_branch2c = mx.symbol.BatchNorm(name='bn3a_branch2c', data=res3a_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3a_branch2c = bn3a_branch2c res3a = mx.symbol.broadcast_add(name='res3a', *[scale3a_branch1,scale3a_branch2c] ) res3a_relu = mx.symbol.Activation(name='res3a_relu', data=res3a , act_type='relu') res3b1_branch2a = mx.symbol.Convolution(name='res3b1_branch2a', data=res3a_relu , num_filter=128, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn3b1_branch2a = mx.symbol.BatchNorm(name='bn3b1_branch2a', data=res3b1_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3b1_branch2a = bn3b1_branch2a res3b1_branch2a_relu = mx.symbol.Activation(name='res3b1_branch2a_relu', data=scale3b1_branch2a , act_type='relu') res3b1_branch2b = mx.symbol.Convolution(name='res3b1_branch2b', data=res3b1_branch2a_relu , num_filter=128, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn3b1_branch2b = mx.symbol.BatchNorm(name='bn3b1_branch2b', data=res3b1_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3b1_branch2b = bn3b1_branch2b res3b1_branch2b_relu = mx.symbol.Activation(name='res3b1_branch2b_relu', data=scale3b1_branch2b , act_type='relu') res3b1_branch2c = mx.symbol.Convolution(name='res3b1_branch2c', data=res3b1_branch2b_relu , num_filter=512, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn3b1_branch2c = mx.symbol.BatchNorm(name='bn3b1_branch2c', data=res3b1_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3b1_branch2c = bn3b1_branch2c res3b1 = mx.symbol.broadcast_add(name='res3b1', *[res3a_relu,scale3b1_branch2c] ) res3b1_relu = mx.symbol.Activation(name='res3b1_relu', data=res3b1 , act_type='relu') res3b2_branch2a = mx.symbol.Convolution(name='res3b2_branch2a', data=res3b1_relu , num_filter=128, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn3b2_branch2a = mx.symbol.BatchNorm(name='bn3b2_branch2a', data=res3b2_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3b2_branch2a = bn3b2_branch2a res3b2_branch2a_relu = mx.symbol.Activation(name='res3b2_branch2a_relu', data=scale3b2_branch2a , act_type='relu') res3b2_branch2b = mx.symbol.Convolution(name='res3b2_branch2b', data=res3b2_branch2a_relu , num_filter=128, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn3b2_branch2b = mx.symbol.BatchNorm(name='bn3b2_branch2b', data=res3b2_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3b2_branch2b = bn3b2_branch2b res3b2_branch2b_relu = mx.symbol.Activation(name='res3b2_branch2b_relu', data=scale3b2_branch2b , act_type='relu') res3b2_branch2c = mx.symbol.Convolution(name='res3b2_branch2c', data=res3b2_branch2b_relu , num_filter=512, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn3b2_branch2c = mx.symbol.BatchNorm(name='bn3b2_branch2c', data=res3b2_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3b2_branch2c = bn3b2_branch2c res3b2 = mx.symbol.broadcast_add(name='res3b2', *[res3b1_relu,scale3b2_branch2c] ) res3b2_relu = mx.symbol.Activation(name='res3b2_relu', data=res3b2 , act_type='relu') res3b3_branch2a = mx.symbol.Convolution(name='res3b3_branch2a', data=res3b2_relu , num_filter=128, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn3b3_branch2a = mx.symbol.BatchNorm(name='bn3b3_branch2a', data=res3b3_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3b3_branch2a = bn3b3_branch2a res3b3_branch2a_relu = mx.symbol.Activation(name='res3b3_branch2a_relu', data=scale3b3_branch2a , act_type='relu') res3b3_branch2b = mx.symbol.Convolution(name='res3b3_branch2b', data=res3b3_branch2a_relu , num_filter=128, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn3b3_branch2b = mx.symbol.BatchNorm(name='bn3b3_branch2b', data=res3b3_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3b3_branch2b = bn3b3_branch2b res3b3_branch2b_relu = mx.symbol.Activation(name='res3b3_branch2b_relu', data=scale3b3_branch2b , act_type='relu') res3b3_branch2c = mx.symbol.Convolution(name='res3b3_branch2c', data=res3b3_branch2b_relu , num_filter=512, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn3b3_branch2c = mx.symbol.BatchNorm(name='bn3b3_branch2c', data=res3b3_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale3b3_branch2c = bn3b3_branch2c res3b3 = mx.symbol.broadcast_add(name='res3b3', *[res3b2_relu,scale3b3_branch2c] ) res3b3_relu = mx.symbol.Activation(name='res3b3_relu', data=res3b3 , act_type='relu') res4a_branch1 = mx.symbol.Convolution(name='res4a_branch1', data=res3b3_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(2,2), no_bias=True) bn4a_branch1 = mx.symbol.BatchNorm(name='bn4a_branch1', data=res4a_branch1 , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4a_branch1 = bn4a_branch1 res4a_branch2a = mx.symbol.Convolution(name='res4a_branch2a', data=res3b3_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(2,2), no_bias=True) bn4a_branch2a = mx.symbol.BatchNorm(name='bn4a_branch2a', data=res4a_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4a_branch2a = bn4a_branch2a res4a_branch2a_relu = mx.symbol.Activation(name='res4a_branch2a_relu', data=scale4a_branch2a , act_type='relu') res4a_branch2b = mx.symbol.Convolution(name='res4a_branch2b', data=res4a_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4a_branch2b = mx.symbol.BatchNorm(name='bn4a_branch2b', data=res4a_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4a_branch2b = bn4a_branch2b res4a_branch2b_relu = mx.symbol.Activation(name='res4a_branch2b_relu', data=scale4a_branch2b , act_type='relu') res4a_branch2c = mx.symbol.Convolution(name='res4a_branch2c', data=res4a_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4a_branch2c = mx.symbol.BatchNorm(name='bn4a_branch2c', data=res4a_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4a_branch2c = bn4a_branch2c res4a = mx.symbol.broadcast_add(name='res4a', *[scale4a_branch1,scale4a_branch2c] ) res4a_relu = mx.symbol.Activation(name='res4a_relu', data=res4a , act_type='relu') res4b1_branch2a = mx.symbol.Convolution(name='res4b1_branch2a', data=res4a_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b1_branch2a = mx.symbol.BatchNorm(name='bn4b1_branch2a', data=res4b1_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b1_branch2a = bn4b1_branch2a res4b1_branch2a_relu = mx.symbol.Activation(name='res4b1_branch2a_relu', data=scale4b1_branch2a , act_type='relu') res4b1_branch2b = mx.symbol.Convolution(name='res4b1_branch2b', data=res4b1_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b1_branch2b = mx.symbol.BatchNorm(name='bn4b1_branch2b', data=res4b1_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b1_branch2b = bn4b1_branch2b res4b1_branch2b_relu = mx.symbol.Activation(name='res4b1_branch2b_relu', data=scale4b1_branch2b , act_type='relu') res4b1_branch2c = mx.symbol.Convolution(name='res4b1_branch2c', data=res4b1_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b1_branch2c = mx.symbol.BatchNorm(name='bn4b1_branch2c', data=res4b1_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b1_branch2c = bn4b1_branch2c res4b1 = mx.symbol.broadcast_add(name='res4b1', *[res4a_relu,scale4b1_branch2c] ) res4b1_relu = mx.symbol.Activation(name='res4b1_relu', data=res4b1 , act_type='relu') res4b2_branch2a = mx.symbol.Convolution(name='res4b2_branch2a', data=res4b1_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b2_branch2a = mx.symbol.BatchNorm(name='bn4b2_branch2a', data=res4b2_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b2_branch2a = bn4b2_branch2a res4b2_branch2a_relu = mx.symbol.Activation(name='res4b2_branch2a_relu', data=scale4b2_branch2a , act_type='relu') res4b2_branch2b = mx.symbol.Convolution(name='res4b2_branch2b', data=res4b2_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b2_branch2b = mx.symbol.BatchNorm(name='bn4b2_branch2b', data=res4b2_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b2_branch2b = bn4b2_branch2b res4b2_branch2b_relu = mx.symbol.Activation(name='res4b2_branch2b_relu', data=scale4b2_branch2b , act_type='relu') res4b2_branch2c = mx.symbol.Convolution(name='res4b2_branch2c', data=res4b2_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b2_branch2c = mx.symbol.BatchNorm(name='bn4b2_branch2c', data=res4b2_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b2_branch2c = bn4b2_branch2c res4b2 = mx.symbol.broadcast_add(name='res4b2', *[res4b1_relu,scale4b2_branch2c] ) res4b2_relu = mx.symbol.Activation(name='res4b2_relu', data=res4b2 , act_type='relu') res4b3_branch2a = mx.symbol.Convolution(name='res4b3_branch2a', data=res4b2_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b3_branch2a = mx.symbol.BatchNorm(name='bn4b3_branch2a', data=res4b3_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b3_branch2a = bn4b3_branch2a res4b3_branch2a_relu = mx.symbol.Activation(name='res4b3_branch2a_relu', data=scale4b3_branch2a , act_type='relu') res4b3_branch2b = mx.symbol.Convolution(name='res4b3_branch2b', data=res4b3_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b3_branch2b = mx.symbol.BatchNorm(name='bn4b3_branch2b', data=res4b3_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b3_branch2b = bn4b3_branch2b res4b3_branch2b_relu = mx.symbol.Activation(name='res4b3_branch2b_relu', data=scale4b3_branch2b , act_type='relu') res4b3_branch2c = mx.symbol.Convolution(name='res4b3_branch2c', data=res4b3_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b3_branch2c = mx.symbol.BatchNorm(name='bn4b3_branch2c', data=res4b3_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b3_branch2c = bn4b3_branch2c res4b3 = mx.symbol.broadcast_add(name='res4b3', *[res4b2_relu,scale4b3_branch2c] ) res4b3_relu = mx.symbol.Activation(name='res4b3_relu', data=res4b3 , act_type='relu') res4b4_branch2a = mx.symbol.Convolution(name='res4b4_branch2a', data=res4b3_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b4_branch2a = mx.symbol.BatchNorm(name='bn4b4_branch2a', data=res4b4_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b4_branch2a = bn4b4_branch2a res4b4_branch2a_relu = mx.symbol.Activation(name='res4b4_branch2a_relu', data=scale4b4_branch2a , act_type='relu') res4b4_branch2b = mx.symbol.Convolution(name='res4b4_branch2b', data=res4b4_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b4_branch2b = mx.symbol.BatchNorm(name='bn4b4_branch2b', data=res4b4_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b4_branch2b = bn4b4_branch2b res4b4_branch2b_relu = mx.symbol.Activation(name='res4b4_branch2b_relu', data=scale4b4_branch2b , act_type='relu') res4b4_branch2c = mx.symbol.Convolution(name='res4b4_branch2c', data=res4b4_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b4_branch2c = mx.symbol.BatchNorm(name='bn4b4_branch2c', data=res4b4_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b4_branch2c = bn4b4_branch2c res4b4 = mx.symbol.broadcast_add(name='res4b4', *[res4b3_relu,scale4b4_branch2c] ) res4b4_relu = mx.symbol.Activation(name='res4b4_relu', data=res4b4 , act_type='relu') res4b5_branch2a = mx.symbol.Convolution(name='res4b5_branch2a', data=res4b4_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b5_branch2a = mx.symbol.BatchNorm(name='bn4b5_branch2a', data=res4b5_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b5_branch2a = bn4b5_branch2a res4b5_branch2a_relu = mx.symbol.Activation(name='res4b5_branch2a_relu', data=scale4b5_branch2a , act_type='relu') res4b5_branch2b = mx.symbol.Convolution(name='res4b5_branch2b', data=res4b5_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b5_branch2b = mx.symbol.BatchNorm(name='bn4b5_branch2b', data=res4b5_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b5_branch2b = bn4b5_branch2b res4b5_branch2b_relu = mx.symbol.Activation(name='res4b5_branch2b_relu', data=scale4b5_branch2b , act_type='relu') res4b5_branch2c = mx.symbol.Convolution(name='res4b5_branch2c', data=res4b5_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b5_branch2c = mx.symbol.BatchNorm(name='bn4b5_branch2c', data=res4b5_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b5_branch2c = bn4b5_branch2c res4b5 = mx.symbol.broadcast_add(name='res4b5', *[res4b4_relu,scale4b5_branch2c] ) res4b5_relu = mx.symbol.Activation(name='res4b5_relu', data=res4b5 , act_type='relu') res4b6_branch2a = mx.symbol.Convolution(name='res4b6_branch2a', data=res4b5_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b6_branch2a = mx.symbol.BatchNorm(name='bn4b6_branch2a', data=res4b6_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b6_branch2a = bn4b6_branch2a res4b6_branch2a_relu = mx.symbol.Activation(name='res4b6_branch2a_relu', data=scale4b6_branch2a , act_type='relu') res4b6_branch2b = mx.symbol.Convolution(name='res4b6_branch2b', data=res4b6_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b6_branch2b = mx.symbol.BatchNorm(name='bn4b6_branch2b', data=res4b6_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b6_branch2b = bn4b6_branch2b res4b6_branch2b_relu = mx.symbol.Activation(name='res4b6_branch2b_relu', data=scale4b6_branch2b , act_type='relu') res4b6_branch2c = mx.symbol.Convolution(name='res4b6_branch2c', data=res4b6_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b6_branch2c = mx.symbol.BatchNorm(name='bn4b6_branch2c', data=res4b6_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b6_branch2c = bn4b6_branch2c res4b6 = mx.symbol.broadcast_add(name='res4b6', *[res4b5_relu,scale4b6_branch2c] ) res4b6_relu = mx.symbol.Activation(name='res4b6_relu', data=res4b6 , act_type='relu') res4b7_branch2a = mx.symbol.Convolution(name='res4b7_branch2a', data=res4b6_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b7_branch2a = mx.symbol.BatchNorm(name='bn4b7_branch2a', data=res4b7_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b7_branch2a = bn4b7_branch2a res4b7_branch2a_relu = mx.symbol.Activation(name='res4b7_branch2a_relu', data=scale4b7_branch2a , act_type='relu') res4b7_branch2b = mx.symbol.Convolution(name='res4b7_branch2b', data=res4b7_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b7_branch2b = mx.symbol.BatchNorm(name='bn4b7_branch2b', data=res4b7_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b7_branch2b = bn4b7_branch2b res4b7_branch2b_relu = mx.symbol.Activation(name='res4b7_branch2b_relu', data=scale4b7_branch2b , act_type='relu') res4b7_branch2c = mx.symbol.Convolution(name='res4b7_branch2c', data=res4b7_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b7_branch2c = mx.symbol.BatchNorm(name='bn4b7_branch2c', data=res4b7_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b7_branch2c = bn4b7_branch2c res4b7 = mx.symbol.broadcast_add(name='res4b7', *[res4b6_relu,scale4b7_branch2c] ) res4b7_relu = mx.symbol.Activation(name='res4b7_relu', data=res4b7 , act_type='relu') res4b8_branch2a = mx.symbol.Convolution(name='res4b8_branch2a', data=res4b7_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b8_branch2a = mx.symbol.BatchNorm(name='bn4b8_branch2a', data=res4b8_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b8_branch2a = bn4b8_branch2a res4b8_branch2a_relu = mx.symbol.Activation(name='res4b8_branch2a_relu', data=scale4b8_branch2a , act_type='relu') res4b8_branch2b = mx.symbol.Convolution(name='res4b8_branch2b', data=res4b8_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b8_branch2b = mx.symbol.BatchNorm(name='bn4b8_branch2b', data=res4b8_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b8_branch2b = bn4b8_branch2b res4b8_branch2b_relu = mx.symbol.Activation(name='res4b8_branch2b_relu', data=scale4b8_branch2b , act_type='relu') res4b8_branch2c = mx.symbol.Convolution(name='res4b8_branch2c', data=res4b8_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b8_branch2c = mx.symbol.BatchNorm(name='bn4b8_branch2c', data=res4b8_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b8_branch2c = bn4b8_branch2c res4b8 = mx.symbol.broadcast_add(name='res4b8', *[res4b7_relu,scale4b8_branch2c] ) res4b8_relu = mx.symbol.Activation(name='res4b8_relu', data=res4b8 , act_type='relu') res4b9_branch2a = mx.symbol.Convolution(name='res4b9_branch2a', data=res4b8_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b9_branch2a = mx.symbol.BatchNorm(name='bn4b9_branch2a', data=res4b9_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b9_branch2a = bn4b9_branch2a res4b9_branch2a_relu = mx.symbol.Activation(name='res4b9_branch2a_relu', data=scale4b9_branch2a , act_type='relu') res4b9_branch2b = mx.symbol.Convolution(name='res4b9_branch2b', data=res4b9_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b9_branch2b = mx.symbol.BatchNorm(name='bn4b9_branch2b', data=res4b9_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b9_branch2b = bn4b9_branch2b res4b9_branch2b_relu = mx.symbol.Activation(name='res4b9_branch2b_relu', data=scale4b9_branch2b , act_type='relu') res4b9_branch2c = mx.symbol.Convolution(name='res4b9_branch2c', data=res4b9_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b9_branch2c = mx.symbol.BatchNorm(name='bn4b9_branch2c', data=res4b9_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b9_branch2c = bn4b9_branch2c res4b9 = mx.symbol.broadcast_add(name='res4b9', *[res4b8_relu,scale4b9_branch2c] ) res4b9_relu = mx.symbol.Activation(name='res4b9_relu', data=res4b9 , act_type='relu') res4b10_branch2a = mx.symbol.Convolution(name='res4b10_branch2a', data=res4b9_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b10_branch2a = mx.symbol.BatchNorm(name='bn4b10_branch2a', data=res4b10_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b10_branch2a = bn4b10_branch2a res4b10_branch2a_relu = mx.symbol.Activation(name='res4b10_branch2a_relu', data=scale4b10_branch2a , act_type='relu') res4b10_branch2b = mx.symbol.Convolution(name='res4b10_branch2b', data=res4b10_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b10_branch2b = mx.symbol.BatchNorm(name='bn4b10_branch2b', data=res4b10_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b10_branch2b = bn4b10_branch2b res4b10_branch2b_relu = mx.symbol.Activation(name='res4b10_branch2b_relu', data=scale4b10_branch2b , act_type='relu') res4b10_branch2c = mx.symbol.Convolution(name='res4b10_branch2c', data=res4b10_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b10_branch2c = mx.symbol.BatchNorm(name='bn4b10_branch2c', data=res4b10_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b10_branch2c = bn4b10_branch2c res4b10 = mx.symbol.broadcast_add(name='res4b10', *[res4b9_relu,scale4b10_branch2c] ) res4b10_relu = mx.symbol.Activation(name='res4b10_relu', data=res4b10 , act_type='relu') res4b11_branch2a = mx.symbol.Convolution(name='res4b11_branch2a', data=res4b10_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b11_branch2a = mx.symbol.BatchNorm(name='bn4b11_branch2a', data=res4b11_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b11_branch2a = bn4b11_branch2a res4b11_branch2a_relu = mx.symbol.Activation(name='res4b11_branch2a_relu', data=scale4b11_branch2a , act_type='relu') res4b11_branch2b = mx.symbol.Convolution(name='res4b11_branch2b', data=res4b11_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b11_branch2b = mx.symbol.BatchNorm(name='bn4b11_branch2b', data=res4b11_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b11_branch2b = bn4b11_branch2b res4b11_branch2b_relu = mx.symbol.Activation(name='res4b11_branch2b_relu', data=scale4b11_branch2b , act_type='relu') res4b11_branch2c = mx.symbol.Convolution(name='res4b11_branch2c', data=res4b11_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b11_branch2c = mx.symbol.BatchNorm(name='bn4b11_branch2c', data=res4b11_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b11_branch2c = bn4b11_branch2c res4b11 = mx.symbol.broadcast_add(name='res4b11', *[res4b10_relu,scale4b11_branch2c] ) res4b11_relu = mx.symbol.Activation(name='res4b11_relu', data=res4b11 , act_type='relu') res4b12_branch2a = mx.symbol.Convolution(name='res4b12_branch2a', data=res4b11_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b12_branch2a = mx.symbol.BatchNorm(name='bn4b12_branch2a', data=res4b12_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b12_branch2a = bn4b12_branch2a res4b12_branch2a_relu = mx.symbol.Activation(name='res4b12_branch2a_relu', data=scale4b12_branch2a , act_type='relu') res4b12_branch2b = mx.symbol.Convolution(name='res4b12_branch2b', data=res4b12_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b12_branch2b = mx.symbol.BatchNorm(name='bn4b12_branch2b', data=res4b12_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b12_branch2b = bn4b12_branch2b res4b12_branch2b_relu = mx.symbol.Activation(name='res4b12_branch2b_relu', data=scale4b12_branch2b , act_type='relu') res4b12_branch2c = mx.symbol.Convolution(name='res4b12_branch2c', data=res4b12_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b12_branch2c = mx.symbol.BatchNorm(name='bn4b12_branch2c', data=res4b12_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b12_branch2c = bn4b12_branch2c res4b12 = mx.symbol.broadcast_add(name='res4b12', *[res4b11_relu,scale4b12_branch2c] ) res4b12_relu = mx.symbol.Activation(name='res4b12_relu', data=res4b12 , act_type='relu') res4b13_branch2a = mx.symbol.Convolution(name='res4b13_branch2a', data=res4b12_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b13_branch2a = mx.symbol.BatchNorm(name='bn4b13_branch2a', data=res4b13_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b13_branch2a = bn4b13_branch2a res4b13_branch2a_relu = mx.symbol.Activation(name='res4b13_branch2a_relu', data=scale4b13_branch2a , act_type='relu') res4b13_branch2b = mx.symbol.Convolution(name='res4b13_branch2b', data=res4b13_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b13_branch2b = mx.symbol.BatchNorm(name='bn4b13_branch2b', data=res4b13_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b13_branch2b = bn4b13_branch2b res4b13_branch2b_relu = mx.symbol.Activation(name='res4b13_branch2b_relu', data=scale4b13_branch2b , act_type='relu') res4b13_branch2c = mx.symbol.Convolution(name='res4b13_branch2c', data=res4b13_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b13_branch2c = mx.symbol.BatchNorm(name='bn4b13_branch2c', data=res4b13_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b13_branch2c = bn4b13_branch2c res4b13 = mx.symbol.broadcast_add(name='res4b13', *[res4b12_relu,scale4b13_branch2c] ) res4b13_relu = mx.symbol.Activation(name='res4b13_relu', data=res4b13 , act_type='relu') res4b14_branch2a = mx.symbol.Convolution(name='res4b14_branch2a', data=res4b13_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b14_branch2a = mx.symbol.BatchNorm(name='bn4b14_branch2a', data=res4b14_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b14_branch2a = bn4b14_branch2a res4b14_branch2a_relu = mx.symbol.Activation(name='res4b14_branch2a_relu', data=scale4b14_branch2a , act_type='relu') res4b14_branch2b = mx.symbol.Convolution(name='res4b14_branch2b', data=res4b14_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b14_branch2b = mx.symbol.BatchNorm(name='bn4b14_branch2b', data=res4b14_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b14_branch2b = bn4b14_branch2b res4b14_branch2b_relu = mx.symbol.Activation(name='res4b14_branch2b_relu', data=scale4b14_branch2b , act_type='relu') res4b14_branch2c = mx.symbol.Convolution(name='res4b14_branch2c', data=res4b14_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b14_branch2c = mx.symbol.BatchNorm(name='bn4b14_branch2c', data=res4b14_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b14_branch2c = bn4b14_branch2c res4b14 = mx.symbol.broadcast_add(name='res4b14', *[res4b13_relu,scale4b14_branch2c] ) res4b14_relu = mx.symbol.Activation(name='res4b14_relu', data=res4b14 , act_type='relu') res4b15_branch2a = mx.symbol.Convolution(name='res4b15_branch2a', data=res4b14_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b15_branch2a = mx.symbol.BatchNorm(name='bn4b15_branch2a', data=res4b15_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b15_branch2a = bn4b15_branch2a res4b15_branch2a_relu = mx.symbol.Activation(name='res4b15_branch2a_relu', data=scale4b15_branch2a , act_type='relu') res4b15_branch2b = mx.symbol.Convolution(name='res4b15_branch2b', data=res4b15_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b15_branch2b = mx.symbol.BatchNorm(name='bn4b15_branch2b', data=res4b15_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b15_branch2b = bn4b15_branch2b res4b15_branch2b_relu = mx.symbol.Activation(name='res4b15_branch2b_relu', data=scale4b15_branch2b , act_type='relu') res4b15_branch2c = mx.symbol.Convolution(name='res4b15_branch2c', data=res4b15_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b15_branch2c = mx.symbol.BatchNorm(name='bn4b15_branch2c', data=res4b15_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b15_branch2c = bn4b15_branch2c res4b15 = mx.symbol.broadcast_add(name='res4b15', *[res4b14_relu,scale4b15_branch2c] ) res4b15_relu = mx.symbol.Activation(name='res4b15_relu', data=res4b15 , act_type='relu') res4b16_branch2a = mx.symbol.Convolution(name='res4b16_branch2a', data=res4b15_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b16_branch2a = mx.symbol.BatchNorm(name='bn4b16_branch2a', data=res4b16_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b16_branch2a = bn4b16_branch2a res4b16_branch2a_relu = mx.symbol.Activation(name='res4b16_branch2a_relu', data=scale4b16_branch2a , act_type='relu') res4b16_branch2b = mx.symbol.Convolution(name='res4b16_branch2b', data=res4b16_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b16_branch2b = mx.symbol.BatchNorm(name='bn4b16_branch2b', data=res4b16_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b16_branch2b = bn4b16_branch2b res4b16_branch2b_relu = mx.symbol.Activation(name='res4b16_branch2b_relu', data=scale4b16_branch2b , act_type='relu') res4b16_branch2c = mx.symbol.Convolution(name='res4b16_branch2c', data=res4b16_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b16_branch2c = mx.symbol.BatchNorm(name='bn4b16_branch2c', data=res4b16_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b16_branch2c = bn4b16_branch2c res4b16 = mx.symbol.broadcast_add(name='res4b16', *[res4b15_relu,scale4b16_branch2c] ) res4b16_relu = mx.symbol.Activation(name='res4b16_relu', data=res4b16 , act_type='relu') res4b17_branch2a = mx.symbol.Convolution(name='res4b17_branch2a', data=res4b16_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b17_branch2a = mx.symbol.BatchNorm(name='bn4b17_branch2a', data=res4b17_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b17_branch2a = bn4b17_branch2a res4b17_branch2a_relu = mx.symbol.Activation(name='res4b17_branch2a_relu', data=scale4b17_branch2a , act_type='relu') res4b17_branch2b = mx.symbol.Convolution(name='res4b17_branch2b', data=res4b17_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b17_branch2b = mx.symbol.BatchNorm(name='bn4b17_branch2b', data=res4b17_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b17_branch2b = bn4b17_branch2b res4b17_branch2b_relu = mx.symbol.Activation(name='res4b17_branch2b_relu', data=scale4b17_branch2b , act_type='relu') res4b17_branch2c = mx.symbol.Convolution(name='res4b17_branch2c', data=res4b17_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b17_branch2c = mx.symbol.BatchNorm(name='bn4b17_branch2c', data=res4b17_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b17_branch2c = bn4b17_branch2c res4b17 = mx.symbol.broadcast_add(name='res4b17', *[res4b16_relu,scale4b17_branch2c] ) res4b17_relu = mx.symbol.Activation(name='res4b17_relu', data=res4b17 , act_type='relu') res4b18_branch2a = mx.symbol.Convolution(name='res4b18_branch2a', data=res4b17_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b18_branch2a = mx.symbol.BatchNorm(name='bn4b18_branch2a', data=res4b18_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b18_branch2a = bn4b18_branch2a res4b18_branch2a_relu = mx.symbol.Activation(name='res4b18_branch2a_relu', data=scale4b18_branch2a , act_type='relu') res4b18_branch2b = mx.symbol.Convolution(name='res4b18_branch2b', data=res4b18_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b18_branch2b = mx.symbol.BatchNorm(name='bn4b18_branch2b', data=res4b18_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b18_branch2b = bn4b18_branch2b res4b18_branch2b_relu = mx.symbol.Activation(name='res4b18_branch2b_relu', data=scale4b18_branch2b , act_type='relu') res4b18_branch2c = mx.symbol.Convolution(name='res4b18_branch2c', data=res4b18_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b18_branch2c = mx.symbol.BatchNorm(name='bn4b18_branch2c', data=res4b18_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b18_branch2c = bn4b18_branch2c res4b18 = mx.symbol.broadcast_add(name='res4b18', *[res4b17_relu,scale4b18_branch2c] ) res4b18_relu = mx.symbol.Activation(name='res4b18_relu', data=res4b18 , act_type='relu') res4b19_branch2a = mx.symbol.Convolution(name='res4b19_branch2a', data=res4b18_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b19_branch2a = mx.symbol.BatchNorm(name='bn4b19_branch2a', data=res4b19_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b19_branch2a = bn4b19_branch2a res4b19_branch2a_relu = mx.symbol.Activation(name='res4b19_branch2a_relu', data=scale4b19_branch2a , act_type='relu') res4b19_branch2b = mx.symbol.Convolution(name='res4b19_branch2b', data=res4b19_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b19_branch2b = mx.symbol.BatchNorm(name='bn4b19_branch2b', data=res4b19_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b19_branch2b = bn4b19_branch2b res4b19_branch2b_relu = mx.symbol.Activation(name='res4b19_branch2b_relu', data=scale4b19_branch2b , act_type='relu') res4b19_branch2c = mx.symbol.Convolution(name='res4b19_branch2c', data=res4b19_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b19_branch2c = mx.symbol.BatchNorm(name='bn4b19_branch2c', data=res4b19_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b19_branch2c = bn4b19_branch2c res4b19 = mx.symbol.broadcast_add(name='res4b19', *[res4b18_relu,scale4b19_branch2c] ) res4b19_relu = mx.symbol.Activation(name='res4b19_relu', data=res4b19 , act_type='relu') res4b20_branch2a = mx.symbol.Convolution(name='res4b20_branch2a', data=res4b19_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b20_branch2a = mx.symbol.BatchNorm(name='bn4b20_branch2a', data=res4b20_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b20_branch2a = bn4b20_branch2a res4b20_branch2a_relu = mx.symbol.Activation(name='res4b20_branch2a_relu', data=scale4b20_branch2a , act_type='relu') res4b20_branch2b = mx.symbol.Convolution(name='res4b20_branch2b', data=res4b20_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b20_branch2b = mx.symbol.BatchNorm(name='bn4b20_branch2b', data=res4b20_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b20_branch2b = bn4b20_branch2b res4b20_branch2b_relu = mx.symbol.Activation(name='res4b20_branch2b_relu', data=scale4b20_branch2b , act_type='relu') res4b20_branch2c = mx.symbol.Convolution(name='res4b20_branch2c', data=res4b20_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b20_branch2c = mx.symbol.BatchNorm(name='bn4b20_branch2c', data=res4b20_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b20_branch2c = bn4b20_branch2c res4b20 = mx.symbol.broadcast_add(name='res4b20', *[res4b19_relu,scale4b20_branch2c] ) res4b20_relu = mx.symbol.Activation(name='res4b20_relu', data=res4b20 , act_type='relu') res4b21_branch2a = mx.symbol.Convolution(name='res4b21_branch2a', data=res4b20_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b21_branch2a = mx.symbol.BatchNorm(name='bn4b21_branch2a', data=res4b21_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b21_branch2a = bn4b21_branch2a res4b21_branch2a_relu = mx.symbol.Activation(name='res4b21_branch2a_relu', data=scale4b21_branch2a , act_type='relu') res4b21_branch2b = mx.symbol.Convolution(name='res4b21_branch2b', data=res4b21_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b21_branch2b = mx.symbol.BatchNorm(name='bn4b21_branch2b', data=res4b21_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b21_branch2b = bn4b21_branch2b res4b21_branch2b_relu = mx.symbol.Activation(name='res4b21_branch2b_relu', data=scale4b21_branch2b , act_type='relu') res4b21_branch2c = mx.symbol.Convolution(name='res4b21_branch2c', data=res4b21_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b21_branch2c = mx.symbol.BatchNorm(name='bn4b21_branch2c', data=res4b21_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b21_branch2c = bn4b21_branch2c res4b21 = mx.symbol.broadcast_add(name='res4b21', *[res4b20_relu,scale4b21_branch2c] ) res4b21_relu = mx.symbol.Activation(name='res4b21_relu', data=res4b21 , act_type='relu') res4b22_branch2a = mx.symbol.Convolution(name='res4b22_branch2a', data=res4b21_relu , num_filter=256, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b22_branch2a = mx.symbol.BatchNorm(name='bn4b22_branch2a', data=res4b22_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b22_branch2a = bn4b22_branch2a res4b22_branch2a_relu = mx.symbol.Activation(name='res4b22_branch2a_relu', data=scale4b22_branch2a , act_type='relu') res4b22_branch2b = mx.symbol.Convolution(name='res4b22_branch2b', data=res4b22_branch2a_relu , num_filter=256, pad=(1,1), kernel=(3,3), stride=(1,1), no_bias=True) bn4b22_branch2b = mx.symbol.BatchNorm(name='bn4b22_branch2b', data=res4b22_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b22_branch2b = bn4b22_branch2b res4b22_branch2b_relu = mx.symbol.Activation(name='res4b22_branch2b_relu', data=scale4b22_branch2b , act_type='relu') res4b22_branch2c = mx.symbol.Convolution(name='res4b22_branch2c', data=res4b22_branch2b_relu , num_filter=1024, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn4b22_branch2c = mx.symbol.BatchNorm(name='bn4b22_branch2c', data=res4b22_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale4b22_branch2c = bn4b22_branch2c res4b22 = mx.symbol.broadcast_add(name='res4b22', *[res4b21_relu,scale4b22_branch2c] ) res4b22_relu = mx.symbol.Activation(name='res4b22_relu', data=res4b22 , act_type='relu') res5a_branch1 = mx.symbol.Convolution(name='res5a_branch1', data=res4b22_relu , num_filter=2048, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn5a_branch1 = mx.symbol.BatchNorm(name='bn5a_branch1', data=res5a_branch1 , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale5a_branch1 = bn5a_branch1 res5a_branch2a = mx.symbol.Convolution(name='res5a_branch2a', data=res4b22_relu , num_filter=512, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn5a_branch2a = mx.symbol.BatchNorm(name='bn5a_branch2a', data=res5a_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale5a_branch2a = bn5a_branch2a res5a_branch2a_relu = mx.symbol.Activation(name='res5a_branch2a_relu', data=scale5a_branch2a , act_type='relu') res5a_branch2b = mx.symbol.Convolution(name='res5a_branch2b', data=res5a_branch2a_relu , num_filter=512, pad=(2,2), dilate=(2,2), kernel=(3,3), stride=(1,1), no_bias=True) bn5a_branch2b = mx.symbol.BatchNorm(name='bn5a_branch2b', data=res5a_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale5a_branch2b = bn5a_branch2b res5a_branch2b_relu = mx.symbol.Activation(name='res5a_branch2b_relu', data=scale5a_branch2b , act_type='relu') res5a_branch2c = mx.symbol.Convolution(name='res5a_branch2c', data=res5a_branch2b_relu , num_filter=2048, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn5a_branch2c = mx.symbol.BatchNorm(name='bn5a_branch2c', data=res5a_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale5a_branch2c = bn5a_branch2c res5a = mx.symbol.broadcast_add(name='res5a', *[scale5a_branch1,scale5a_branch2c] ) res5a_relu = mx.symbol.Activation(name='res5a_relu', data=res5a , act_type='relu') res5b_branch2a = mx.symbol.Convolution(name='res5b_branch2a', data=res5a_relu , num_filter=512, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn5b_branch2a = mx.symbol.BatchNorm(name='bn5b_branch2a', data=res5b_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale5b_branch2a = bn5b_branch2a res5b_branch2a_relu = mx.symbol.Activation(name='res5b_branch2a_relu', data=scale5b_branch2a , act_type='relu') res5b_branch2b = mx.symbol.Convolution(name='res5b_branch2b', data=res5b_branch2a_relu , num_filter=512, pad=(2,2), dilate=(2,2), kernel=(3,3), stride=(1,1), no_bias=True) bn5b_branch2b = mx.symbol.BatchNorm(name='bn5b_branch2b', data=res5b_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale5b_branch2b = bn5b_branch2b res5b_branch2b_relu = mx.symbol.Activation(name='res5b_branch2b_relu', data=scale5b_branch2b , act_type='relu') res5b_branch2c = mx.symbol.Convolution(name='res5b_branch2c', data=res5b_branch2b_relu , num_filter=2048, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn5b_branch2c = mx.symbol.BatchNorm(name='bn5b_branch2c', data=res5b_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale5b_branch2c = bn5b_branch2c res5b = mx.symbol.broadcast_add(name='res5b', *[res5a_relu,scale5b_branch2c] ) res5b_relu = mx.symbol.Activation(name='res5b_relu', data=res5b , act_type='relu') res5c_branch2a = mx.symbol.Convolution(name='res5c_branch2a', data=res5b_relu , num_filter=512, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn5c_branch2a = mx.symbol.BatchNorm(name='bn5c_branch2a', data=res5c_branch2a , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale5c_branch2a = bn5c_branch2a res5c_branch2a_relu = mx.symbol.Activation(name='res5c_branch2a_relu', data=scale5c_branch2a , act_type='relu') res5c_branch2b = mx.symbol.Convolution(name='res5c_branch2b', data=res5c_branch2a_relu , num_filter=512, pad=(2,2), dilate=(2,2), kernel=(3,3), stride=(1,1), no_bias=True) bn5c_branch2b = mx.symbol.BatchNorm(name='bn5c_branch2b', data=res5c_branch2b , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale5c_branch2b = bn5c_branch2b res5c_branch2b_relu = mx.symbol.Activation(name='res5c_branch2b_relu', data=scale5c_branch2b , act_type='relu') res5c_branch2c = mx.symbol.Convolution(name='res5c_branch2c', data=res5c_branch2b_relu , num_filter=2048, pad=(0,0), kernel=(1,1), stride=(1,1), no_bias=True) bn5c_branch2c = mx.symbol.BatchNorm(name='bn5c_branch2c', data=res5c_branch2c , use_global_stats=self.use_global_stats, eps=self.eps, fix_gamma=False) scale5c_branch2c = bn5c_branch2c res5c = mx.symbol.broadcast_add(name='res5c', *[res5b_relu,scale5c_branch2c] ) res5c_relu = mx.symbol.Activation(name='res5c_relu', data=res5c , act_type='relu') feat_conv_3x3 = mx.sym.Convolution( data=res5c_relu, kernel=(3, 3), pad=(6, 6), dilate=(6, 6), num_filter=1024, name="feat_conv_3x3") feat_conv_3x3_relu = mx.sym.Activation(data=feat_conv_3x3, act_type="relu", name="feat_conv_3x3_relu") return feat_conv_3x3_relu def get_train_symbol(self, cfg): # config alias for convenient num_classes = cfg.dataset.NUM_CLASSES num_reg_classes = (2 if cfg.CLASS_AGNOSTIC else num_classes) num_anchors = cfg.network.NUM_ANCHORS data = mx.sym.Variable(name="data") im_info = mx.sym.Variable(name="im_info") gt_boxes = mx.sym.Variable(name="gt_boxes") rpn_label = mx.sym.Variable(name='label') rpn_bbox_target = mx.sym.Variable(name='bbox_target') rpn_bbox_weight = mx.sym.Variable(name='bbox_weight') # shared convolutional layers conv_feat = self.get_resnet_v1(data) conv_feats = mx.sym.SliceChannel(conv_feat, axis=1, num_outputs=2) # RPN layers rpn_feat = conv_feats[0] rpn_cls_score = mx.sym.Convolution( data=rpn_feat, kernel=(1, 1), pad=(0, 0), num_filter=2 * num_anchors, name="rpn_cls_score") rpn_bbox_pred = mx.sym.Convolution( data=rpn_feat, kernel=(1, 1), pad=(0, 0), num_filter=4 * num_anchors, name="rpn_bbox_pred") # prepare rpn data rpn_cls_score_reshape = mx.sym.Reshape( data=rpn_cls_score, shape=(0, 2, -1, 0), name="rpn_cls_score_reshape") # classification rpn_cls_prob = mx.sym.SoftmaxOutput(data=rpn_cls_score_reshape, label=rpn_label, multi_output=True, normalization='valid', use_ignore=True, ignore_label=-1, name="rpn_cls_prob") # bounding box regression if cfg.network.NORMALIZE_RPN: rpn_bbox_loss_ = rpn_bbox_weight * mx.sym.smooth_l1(name='rpn_bbox_loss_', scalar=1.0, data=(rpn_bbox_pred - rpn_bbox_target)) rpn_bbox_pred = mx.sym.Custom( bbox_pred=rpn_bbox_pred, op_type='rpn_inv_normalize', num_anchors=num_anchors, bbox_mean=cfg.network.ANCHOR_MEANS, bbox_std=cfg.network.ANCHOR_STDS) else: rpn_bbox_loss_ = rpn_bbox_weight * mx.sym.smooth_l1(name='rpn_bbox_loss_', scalar=3.0, data=(rpn_bbox_pred - rpn_bbox_target)) rpn_bbox_loss = mx.sym.MakeLoss(name='rpn_bbox_loss', data=rpn_bbox_loss_, grad_scale=1.0 / cfg.TRAIN.RPN_BATCH_SIZE) # ROI proposal rpn_cls_act = mx.sym.SoftmaxActivation( data=rpn_cls_score_reshape, mode="channel", name="rpn_cls_act") rpn_cls_act_reshape = mx.sym.Reshape( data=rpn_cls_act, shape=(0, 2 * num_anchors, -1, 0), name='rpn_cls_act_reshape') if cfg.TRAIN.CXX_PROPOSAL: rois = mx.contrib.sym.Proposal( cls_prob=rpn_cls_act_reshape, bbox_pred=rpn_bbox_pred, im_info=im_info, name='rois', feature_stride=cfg.network.RPN_FEAT_STRIDE, scales=tuple(cfg.network.ANCHOR_SCALES), ratios=tuple(cfg.network.ANCHOR_RATIOS), rpn_pre_nms_top_n=cfg.TRAIN.RPN_PRE_NMS_TOP_N, rpn_post_nms_top_n=cfg.TRAIN.RPN_POST_NMS_TOP_N, threshold=cfg.TRAIN.RPN_NMS_THRESH, rpn_min_size=cfg.TRAIN.RPN_MIN_SIZE) else: rois = mx.sym.Custom( cls_prob=rpn_cls_act_reshape, bbox_pred=rpn_bbox_pred, im_info=im_info, name='rois', op_type='proposal', feat_stride=cfg.network.RPN_FEAT_STRIDE, scales=tuple(cfg.network.ANCHOR_SCALES), ratios=tuple(cfg.network.ANCHOR_RATIOS), rpn_pre_nms_top_n=cfg.TRAIN.RPN_PRE_NMS_TOP_N, rpn_post_nms_top_n=cfg.TRAIN.RPN_POST_NMS_TOP_N, threshold=cfg.TRAIN.RPN_NMS_THRESH, rpn_min_size=cfg.TRAIN.RPN_MIN_SIZE) # ROI proposal target gt_boxes_reshape = mx.sym.Reshape(data=gt_boxes, shape=(-1, 5), name='gt_boxes_reshape') rois, label, bbox_target, bbox_weight = mx.sym.Custom(rois=rois, gt_boxes=gt_boxes_reshape, op_type='proposal_target', num_classes=num_reg_classes, batch_images=cfg.TRAIN.BATCH_IMAGES, batch_rois=cfg.TRAIN.BATCH_ROIS, cfg=cPickle.dumps(cfg), fg_fraction=cfg.TRAIN.FG_FRACTION) # res5 rfcn_feat = conv_feats[1] rfcn_cls = mx.sym.Convolution(data=rfcn_feat, kernel=(1, 1), num_filter=7*7*num_classes, name="rfcn_cls") rfcn_bbox = mx.sym.Convolution(data=rfcn_feat, kernel=(1, 1), num_filter=7*7*4*num_reg_classes, name="rfcn_bbox") psroipooled_cls_rois = mx.contrib.sym.PSROIPooling(name='psroipooled_cls_rois', data=rfcn_cls, rois=rois, group_size=7,pooled_size=7, output_dim=num_classes, spatial_scale=0.0625) psroipooled_loc_rois = mx.contrib.sym.PSROIPooling(name='psroipooled_loc_rois', data=rfcn_bbox, rois=rois, group_size=7,pooled_size=7, output_dim=8, spatial_scale=0.0625) cls_score = mx.sym.Pooling(name='ave_cls_scors_rois', data=psroipooled_cls_rois, pool_type='avg', global_pool=True, kernel=(7, 7)) bbox_pred = mx.sym.Pooling(name='ave_bbox_pred_rois', data=psroipooled_loc_rois, pool_type='avg', global_pool=True, kernel=(7, 7)) cls_score = mx.sym.Reshape(name='cls_score_reshape', data=cls_score, shape=(-1, num_classes)) bbox_pred = mx.sym.Reshape(name='bbox_pred_reshape', data=bbox_pred, shape=(-1, 4 * num_reg_classes)) # classification if cfg.TRAIN.ENABLE_OHEM: print 'use ohem!' labels_ohem, bbox_weights_ohem = mx.sym.Custom(op_type='BoxAnnotatorOHEM', num_classes=num_classes, num_reg_classes=num_reg_classes, roi_per_img=cfg.TRAIN.BATCH_ROIS_OHEM, cls_score=cls_score, bbox_pred=bbox_pred, labels=label, bbox_targets=bbox_target, bbox_weights=bbox_weight) cls_prob = mx.sym.SoftmaxOutput(name='cls_prob', data=cls_score, label=labels_ohem, normalization='valid', use_ignore=True, ignore_label=-1) bbox_loss_ = bbox_weights_ohem * mx.sym.smooth_l1(name='bbox_loss_', scalar=1.0, data=(bbox_pred - bbox_target)) bbox_loss = mx.sym.MakeLoss(name='bbox_loss', data=bbox_loss_, grad_scale=1.0 / cfg.TRAIN.BATCH_ROIS_OHEM) rcnn_label = labels_ohem else: cls_prob = mx.sym.SoftmaxOutput(name='cls_prob', data=cls_score, label=label, normalization='valid') bbox_loss_ = bbox_weight * mx.sym.smooth_l1(name='bbox_loss_', scalar=1.0, data=(bbox_pred - bbox_target)) bbox_loss = mx.sym.MakeLoss(name='bbox_loss', data=bbox_loss_, grad_scale=1.0 / cfg.TRAIN.BATCH_ROIS) rcnn_label = label # reshape output rcnn_label = mx.sym.Reshape(data=rcnn_label, shape=(cfg.TRAIN.BATCH_IMAGES, -1), name='label_reshape') cls_prob = mx.sym.Reshape(data=cls_prob, shape=(cfg.TRAIN.BATCH_IMAGES, -1, num_classes), name='cls_prob_reshape') bbox_loss = mx.sym.Reshape(data=bbox_loss, shape=(cfg.TRAIN.BATCH_IMAGES, -1, 4 * num_reg_classes), name='bbox_loss_reshape') group = mx.sym.Group([rpn_cls_prob, rpn_bbox_loss, cls_prob, bbox_loss, mx.sym.BlockGrad(rcnn_label)]) self.sym = group return group def get_test_symbol(self, cfg): # config alias for convenient num_classes = cfg.dataset.NUM_CLASSES num_reg_classes = (2 if cfg.CLASS_AGNOSTIC else num_classes) num_anchors = cfg.network.NUM_ANCHORS data = mx.sym.Variable(name="data") im_info = mx.sym.Variable(name="im_info") # shared convolutional layers conv_feat = self.get_resnet_v1(data) conv_feats = mx.sym.SliceChannel(conv_feat, axis=1, num_outputs=2) # RPN rpn_feat = conv_feats[0] rpn_cls_score = mx.sym.Convolution( data=rpn_feat, kernel=(1, 1), pad=(0, 0), num_filter=2 * num_anchors, name="rpn_cls_score") rpn_bbox_pred = mx.sym.Convolution( data=rpn_feat, kernel=(1, 1), pad=(0, 0), num_filter=4 * num_anchors, name="rpn_bbox_pred") if cfg.network.NORMALIZE_RPN: rpn_bbox_pred = mx.sym.Custom( bbox_pred=rpn_bbox_pred, op_type='rpn_inv_normalize', num_anchors=num_anchors, bbox_mean=cfg.network.ANCHOR_MEANS, bbox_std=cfg.network.ANCHOR_STDS) # ROI Proposal rpn_cls_score_reshape = mx.sym.Reshape( data=rpn_cls_score, shape=(0, 2, -1, 0), name="rpn_cls_score_reshape") rpn_cls_prob = mx.sym.SoftmaxActivation( data=rpn_cls_score_reshape, mode="channel", name="rpn_cls_prob") rpn_cls_prob_reshape = mx.sym.Reshape( data=rpn_cls_prob, shape=(0, 2 * num_anchors, -1, 0), name='rpn_cls_prob_reshape') if cfg.TEST.CXX_PROPOSAL: rois = mx.contrib.sym.MultiProposal( cls_prob=rpn_cls_prob_reshape, bbox_pred=rpn_bbox_pred, im_info=im_info, name='rois', feature_stride=cfg.network.RPN_FEAT_STRIDE, scales=tuple(cfg.network.ANCHOR_SCALES), ratios=tuple(cfg.network.ANCHOR_RATIOS), rpn_pre_nms_top_n=cfg.TEST.RPN_PRE_NMS_TOP_N, rpn_post_nms_top_n=cfg.TEST.RPN_POST_NMS_TOP_N, threshold=cfg.TEST.RPN_NMS_THRESH, rpn_min_size=cfg.TEST.RPN_MIN_SIZE) else: NotImplemented # res5 rfcn_feat = conv_feats[1] rfcn_cls = mx.sym.Convolution(data=rfcn_feat, kernel=(1, 1), num_filter=7*7*num_classes, name="rfcn_cls") rfcn_bbox = mx.sym.Convolution(data=rfcn_feat, kernel=(1, 1), num_filter=7*7*4*num_reg_classes, name="rfcn_bbox") psroipooled_cls_rois = mx.contrib.sym.PSROIPooling(name='psroipooled_cls_rois', data=rfcn_cls, rois=rois, group_size=7, pooled_size=7, output_dim=num_classes, spatial_scale=0.0625) psroipooled_loc_rois = mx.contrib.sym.PSROIPooling(name='psroipooled_loc_rois', data=rfcn_bbox, rois=rois, group_size=7, pooled_size=7, output_dim=8, spatial_scale=0.0625) cls_score = mx.sym.Pooling(name='ave_cls_scors_rois', data=psroipooled_cls_rois, pool_type='avg', global_pool=True, kernel=(7, 7)) bbox_pred = mx.sym.Pooling(name='ave_bbox_pred_rois', data=psroipooled_loc_rois, pool_type='avg', global_pool=True, kernel=(7, 7)) # classification cls_score = mx.sym.Reshape(name='cls_score_reshape', data=cls_score, shape=(-1, num_classes)) cls_prob = mx.sym.SoftmaxActivation(name='cls_prob', data=cls_score) # bounding box regression bbox_pred = mx.sym.Reshape(name='bbox_pred_reshape', data=bbox_pred, shape=(-1, 4 * num_reg_classes)) # reshape output cls_prob = mx.sym.Reshape(data=cls_prob, shape=(cfg.TEST.BATCH_IMAGES, -1, num_classes), name='cls_prob_reshape') bbox_pred = mx.sym.Reshape(data=bbox_pred, shape=(cfg.TEST.BATCH_IMAGES, -1, 4 * num_reg_classes), name='bbox_pred_reshape') # group output group = mx.sym.Group([rois, cls_prob, bbox_pred]) self.sym = group return group def init_weight(self, cfg, arg_params, aux_params): arg_params['feat_conv_3x3_weight'] = mx.random.normal(0, 0.01, shape=self.arg_shape_dict['feat_conv_3x3_weight']) arg_params['feat_conv_3x3_bias'] = mx.nd.zeros(shape=self.arg_shape_dict['feat_conv_3x3_bias']) arg_params['rpn_cls_score_weight'] = mx.random.normal(0, 0.01, shape=self.arg_shape_dict['rpn_cls_score_weight']) arg_params['rpn_cls_score_bias'] = mx.nd.zeros(shape=self.arg_shape_dict['rpn_cls_score_bias']) arg_params['rpn_bbox_pred_weight'] = mx.random.normal(0, 0.01, shape=self.arg_shape_dict['rpn_bbox_pred_weight']) arg_params['rpn_bbox_pred_bias'] = mx.nd.zeros(shape=self.arg_shape_dict['rpn_bbox_pred_bias']) arg_params['rfcn_cls_weight'] = mx.random.normal(0, 0.01, shape=self.arg_shape_dict['rfcn_cls_weight']) arg_params['rfcn_cls_bias'] = mx.nd.zeros(shape=self.arg_shape_dict['rfcn_cls_bias']) arg_params['rfcn_bbox_weight'] = mx.random.normal(0, 0.01, shape=self.arg_shape_dict['rfcn_bbox_weight']) arg_params['rfcn_bbox_bias'] = mx.nd.zeros(shape=self.arg_shape_dict['rfcn_bbox_bias']) ================================================ FILE: rfcn/test.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- import _init_paths import cv2 import argparse import os import sys import time import logging from config.config import config, update_config def parse_args(): parser = argparse.ArgumentParser(description='Test a R-FCN network') # general parser.add_argument('--cfg', help='experiment configure file name', required=True, type=str) args, rest = parser.parse_known_args() update_config(args.cfg) # rcnn parser.add_argument('--vis', help='turn on visualization', action='store_true') parser.add_argument('--ignore_cache', help='ignore cached results boxes', action='store_true') parser.add_argument('--thresh', help='valid detection threshold', default=1e-3, type=float) parser.add_argument('--shuffle', help='shuffle data on visualization', action='store_true') args = parser.parse_args() return args args = parse_args() curr_path = os.path.abspath(os.path.dirname(__file__)) sys.path.insert(0, os.path.join(curr_path, '../external/mxnet', config.MXNET_VERSION)) import mxnet as mx from function.test_rcnn import test_rcnn from utils.create_logger import create_logger def main(): ctx = [mx.gpu(int(i)) for i in config.gpus.split(',')] print args logger, final_output_path = create_logger(config.output_path, args.cfg, config.dataset.test_image_set) test_rcnn(config, config.dataset.dataset, config.dataset.test_image_set, config.dataset.root_path, config.dataset.dataset_path, ctx, os.path.join(final_output_path, '..', '_'.join([iset for iset in config.dataset.image_set.split('+')]), config.TRAIN.model_prefix), config.TEST.test_epoch, args.vis, args.ignore_cache, args.shuffle, config.TEST.HAS_RPN, config.dataset.proposal, args.thresh, logger=logger, output_path=final_output_path) if __name__ == '__main__': main() ================================================ FILE: rfcn/train_end2end.py ================================================ # -------------------------------------------------------- # Deep Feature Flow # Copyright (c) 2017 Microsoft # Licensed under The MIT License [see LICENSE for details] # Modified by Yuwen Xiong # -------------------------------------------------------- # Based on: # MX-RCNN # Copyright (c) 2016 by Contributors # Licence under The Apache 2.0 License # https://github.com/ijkguo/mx-rcnn/ # -------------------------------------------------------- import _init_paths import cv2 import time import argparse import logging import pprint import os import sys from config.config import config, update_config def parse_args(): parser = argparse.ArgumentParser(description='Train R-FCN network') # general parser.add_argument('--cfg', help='experiment configure file name', required=True, type=str) args, rest = parser.parse_known_args() # update config update_config(args.cfg) # training parser.add_argument('--frequent', help='frequency of logging', default=config.default.frequent, type=int) args = parser.parse_args() return args args = parse_args() curr_path = os.path.abspath(os.path.dirname(__file__)) sys.path.insert(0, os.path.join(curr_path, '../external/mxnet', config.MXNET_VERSION)) import shutil import numpy as np import mxnet as mx from symbols import * from core import callback, metric from core.loader import AnchorLoader from core.module import MutableModule from utils.create_logger import create_logger from utils.load_data import load_gt_roidb, merge_roidb, filter_roidb from utils.load_model import load_param from utils.PrefetchingIter import PrefetchingIter from utils.lr_scheduler import WarmupMultiFactorScheduler def train_net(args, ctx, pretrained, epoch, prefix, begin_epoch, end_epoch, lr, lr_step): logger, final_output_path = create_logger(config.output_path, args.cfg, config.dataset.image_set) prefix = os.path.join(final_output_path, prefix) # load symbol shutil.copy2(os.path.join(curr_path, 'symbols', config.symbol + '.py'), final_output_path) sym_instance = eval(config.symbol + '.' + config.symbol)() sym = sym_instance.get_train_symbol(config) feat_sym = sym.get_internals()['rpn_cls_score_output'] # setup multi-gpu batch_size = len(ctx) input_batch_size = config.TRAIN.BATCH_IMAGES * batch_size # print config pprint.pprint(config) logger.info('training config:{}\n'.format(pprint.pformat(config))) # load dataset and prepare imdb for training image_sets = [iset for iset in config.dataset.image_set.split('+')] roidbs = [load_gt_roidb(config.dataset.dataset, image_set, config.dataset.root_path, config.dataset.dataset_path, flip=config.TRAIN.FLIP) for image_set in image_sets] roidb = merge_roidb(roidbs) roidb = filter_roidb(roidb, config) # load training data train_data = AnchorLoader(feat_sym, roidb, config, batch_size=input_batch_size, shuffle=config.TRAIN.SHUFFLE, ctx=ctx, feat_stride=config.network.RPN_FEAT_STRIDE, anchor_scales=config.network.ANCHOR_SCALES, anchor_ratios=config.network.ANCHOR_RATIOS, aspect_grouping=config.TRAIN.ASPECT_GROUPING, normalize_target=config.network.NORMALIZE_RPN, bbox_mean=config.network.ANCHOR_MEANS, bbox_std=config.network.ANCHOR_STDS) # infer max shape max_data_shape = [('data', (config.TRAIN.BATCH_IMAGES, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))] max_data_shape, max_label_shape = train_data.infer_shape(max_data_shape) max_data_shape.append(('gt_boxes', (config.TRAIN.BATCH_IMAGES, 100, 5))) print('providing maximum shape', max_data_shape, max_label_shape) data_shape_dict = dict(train_data.provide_data_single + train_data.provide_label_single) pprint.pprint(data_shape_dict) sym_instance.infer_shape(data_shape_dict) # load and initialize params if config.TRAIN.RESUME: print('continue training from ', begin_epoch) arg_params, aux_params = load_param(prefix, begin_epoch, convert=True) else: arg_params, aux_params = load_param(pretrained, epoch, convert=True) sym_instance.init_weight(config, arg_params, aux_params) # check parameter shapes sym_instance.check_parameter_shapes(arg_params, aux_params, data_shape_dict) # create solver fixed_param_prefix = config.network.FIXED_PARAMS data_names = [k[0] for k in train_data.provide_data_single] label_names = [k[0] for k in train_data.provide_label_single] mod = MutableModule(sym, data_names=data_names, label_names=label_names, logger=logger, context=ctx, max_data_shapes=[max_data_shape for _ in range(batch_size)], max_label_shapes=[max_label_shape for _ in range(batch_size)], fixed_param_prefix=fixed_param_prefix) if config.TRAIN.RESUME: mod._preload_opt_states = '%s-%04d.states'%(prefix, begin_epoch) # decide training params # metric rpn_eval_metric = metric.RPNAccMetric() rpn_cls_metric = metric.RPNLogLossMetric() rpn_bbox_metric = metric.RPNL1LossMetric() eval_metric = metric.RCNNAccMetric(config) cls_metric = metric.RCNNLogLossMetric(config) bbox_metric = metric.RCNNL1LossMetric(config) eval_metrics = mx.metric.CompositeEvalMetric() # rpn_eval_metric, rpn_cls_metric, rpn_bbox_metric, eval_metric, cls_metric, bbox_metric for child_metric in [rpn_eval_metric, rpn_cls_metric, rpn_bbox_metric, eval_metric, cls_metric, bbox_metric]: eval_metrics.add(child_metric) # callback batch_end_callback = callback.Speedometer(train_data.batch_size, frequent=args.frequent) means = np.tile(np.array(config.TRAIN.BBOX_MEANS), 2 if config.CLASS_AGNOSTIC else config.dataset.NUM_CLASSES) stds = np.tile(np.array(config.TRAIN.BBOX_STDS), 2 if config.CLASS_AGNOSTIC else config.dataset.NUM_CLASSES) epoch_end_callback = [mx.callback.module_checkpoint(mod, prefix, period=1, save_optimizer_states=True), callback.do_checkpoint(prefix, means, stds)] # decide learning rate base_lr = lr lr_factor = config.TRAIN.lr_factor lr_epoch = [float(epoch) for epoch in lr_step.split(',')] lr_epoch_diff = [epoch - begin_epoch for epoch in lr_epoch if epoch > begin_epoch] lr = base_lr * (lr_factor ** (len(lr_epoch) - len(lr_epoch_diff))) lr_iters = [int(epoch * len(roidb) / batch_size) for epoch in lr_epoch_diff] print('lr', lr, 'lr_epoch_diff', lr_epoch_diff, 'lr_iters', lr_iters) lr_scheduler = WarmupMultiFactorScheduler(lr_iters, lr_factor, config.TRAIN.warmup, config.TRAIN.warmup_lr, config.TRAIN.warmup_step) # optimizer optimizer_params = {'momentum': config.TRAIN.momentum, 'wd': config.TRAIN.wd, 'learning_rate': lr, 'lr_scheduler': lr_scheduler, 'rescale_grad': 1.0, 'clip_gradient': None} if not isinstance(train_data, PrefetchingIter): train_data = PrefetchingIter(train_data) # train mod.fit(train_data, eval_metric=eval_metrics, epoch_end_callback=epoch_end_callback, batch_end_callback=batch_end_callback, kvstore=config.default.kvstore, optimizer='sgd', optimizer_params=optimizer_params, arg_params=arg_params, aux_params=aux_params, begin_epoch=begin_epoch, num_epoch=end_epoch) def main(): print('Called with argument:', args) ctx = [mx.gpu(int(i)) for i in config.gpus.split(',')] train_net(args, ctx, config.network.pretrained, config.network.pretrained_epoch, config.TRAIN.model_prefix, config.TRAIN.begin_epoch, config.TRAIN.end_epoch, config.TRAIN.lr, config.TRAIN.lr_step) if __name__ == '__main__': main() ================================================ FILE: zero.md ================================================ 0