[ { "path": "CMakeLists.txt", "content": "cmake_minimum_required(VERSION 2.8.3)\nproject(jpda_rospack)\n\n## Compile as C++11, supported in ROS Kinetic and newer\n# add_compile_options(-std=c++11)\n\n## Find catkin macros and libraries\n## if COMPONENTS list like find_package(catkin REQUIRED COMPONENTS xyz)\n## is used, also find other catkin packages\nfind_package(catkin REQUIRED COMPONENTS\n roscpp\n rospy\n std_msgs\n vision_msgs\n message_generation\n)\n\n## System dependencies are found with CMake's conventions\n# find_package(Boost REQUIRED COMPONENTS system)\n\n\n## Uncomment this if the package has a setup.py. This macro ensures\n## modules and global scripts declared therein get installed\n## See http://ros.org/doc/api/catkin/html/user_guide/setup_dot_py.html\n# catkin_python_setup()\n\n################################################\n## Declare ROS messages, services and actions ##\n################################################\n\n## To declare and build messages, services or actions from within this\n## package, follow these steps:\n## * Let MSG_DEP_SET be the set of packages whose message types you use in\n## your messages/services/actions (e.g. std_msgs, actionlib_msgs, ...).\n## * In the file package.xml:\n## * add a build_depend tag for \"message_generation\"\n## * add a build_depend and a exec_depend tag for each package in MSG_DEP_SET\n## * If MSG_DEP_SET isn't empty the following dependency has been pulled in\n## but can be declared for certainty nonetheless:\n## * add a exec_depend tag for \"message_runtime\"\n## * In this file (CMakeLists.txt):\n## * add \"message_generation\" and every package in MSG_DEP_SET to\n## find_package(catkin REQUIRED COMPONENTS ...)\n## * add \"message_runtime\" and every package in MSG_DEP_SET to\n## catkin_package(CATKIN_DEPENDS ...)\n## * uncomment the add_*_files sections below as needed\n## and list every .msg/.srv/.action file to be processed\n## * uncomment the generate_messages entry below\n## * add every package in MSG_DEP_SET to generate_messages(DEPENDENCIES ...)\n\n## Generate messages in the 'msg' folder\nadd_message_files(\n FILES\n detection2d_with_feature.msg\n detection2d_with_feature_array.msg\n detection3d_with_feature.msg\n detection3d_with_feature_array.msg\n )\n\n## Generate services in the 'srv' folder\n# add_service_files(\n# FILES\n# Service1.srv\n# Service2.srv\n# )\n\n## Generate actions in the 'action' folder\n# add_action_files(\n# FILES\n# Action1.action\n# Action2.action\n# )\n\n## Generate added messages and services with any dependencies listed here\n generate_messages(\n DEPENDENCIES\n std_msgs\n vision_msgs\n jpda_rospack\n )\n\n################################################\n## Declare ROS dynamic reconfigure parameters ##\n################################################\n\n## To declare and build dynamic reconfigure parameters within this\n## package, follow these steps:\n## * In the file package.xml:\n## * add a build_depend and a exec_depend tag for \"dynamic_reconfigure\"\n## * In this file (CMakeLists.txt):\n## * add \"dynamic_reconfigure\" to\n## find_package(catkin REQUIRED COMPONENTS ...)\n## * uncomment the \"generate_dynamic_reconfigure_options\" section below\n## and list every .cfg file to be processed\n\n## Generate dynamic reconfigure parameters in the 'cfg' folder\n# generate_dynamic_reconfigure_options(\n# cfg/DynReconf1.cfg\n# cfg/DynReconf2.cfg\n# )\n\n###################################\n## catkin specific configuration ##\n###################################\n## The catkin_package macro generates cmake config files for your package\n## Declare things to be passed to dependent projects\n## INCLUDE_DIRS: uncomment this if your package contains header files\n## LIBRARIES: libraries you create in this project that dependent projects also need\n## CATKIN_DEPENDS: catkin_packages dependent projects also need\n## DEPENDS: system dependencies of this project that dependent projects also need\ncatkin_package(\n# INCLUDE_DIRS include\n# LIBRARIES jpda_rospack\n CATKIN_DEPENDS roscpp rospy std_msgs vision_msgs message_runtime\n# DEPENDS system_lib\n)\n\n###########\n## Build ##\n###########\n\n## Specify additional locations of header files\n## Your package locations should be listed before other locations\ninclude_directories(\n# include\n ${catkin_INCLUDE_DIRS}\n)\n\n## Declare a C++ library\n# add_library(${PROJECT_NAME}\n# src/${PROJECT_NAME}/jpda_rospack.cpp\n# )\n\n## Add cmake target dependencies of the library\n## as an example, code may need to be generated before libraries\n## either from message generation or dynamic reconfigure\n# add_dependencies(${PROJECT_NAME} ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS})\n\n## Declare a C++ executable\n## With catkin_make all packages are built within a single CMake context\n## The recommended prefix ensures that target names across packages don't collide\n# add_executable(${PROJECT_NAME}_node src/jpda_rospack_node.cpp)\n\n## Rename C++ executable without prefix\n## The above recommended prefix causes long target names, the following renames the\n## target back to the shorter version for ease of user use\n## e.g. \"rosrun someones_pkg node\" instead of \"rosrun someones_pkg someones_pkg_node\"\n# set_target_properties(${PROJECT_NAME}_node PROPERTIES OUTPUT_NAME node PREFIX \"\")\n\n## Add cmake target dependencies of the executable\n## same as for the library above\n# add_dependencies(${PROJECT_NAME}_node ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS})\n\n## Specify libraries to link a library or executable target against\n# target_link_libraries(${PROJECT_NAME}_node\n# ${catkin_LIBRARIES}\n# )\n\n#############\n## Install ##\n#############\n\n# all install targets should use catkin DESTINATION variables\n# See http://ros.org/doc/api/catkin/html/adv_user_guide/variables.html\n\n## Mark executable scripts (Python etc.) for installation\n## in contrast to setup.py, you can choose the destination\n# install(PROGRAMS\n# scripts/my_python_script\n# DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION}\n# )\n\n## Mark executables and/or libraries for installation\n# install(TARGETS ${PROJECT_NAME} ${PROJECT_NAME}_node\n# ARCHIVE DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION}\n# LIBRARY DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION}\n# RUNTIME DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION}\n# )\n\n## Mark cpp header files for installation\n# install(DIRECTORY include/${PROJECT_NAME}/\n# DESTINATION ${CATKIN_PACKAGE_INCLUDE_DESTINATION}\n# FILES_MATCHING PATTERN \"*.h\"\n# PATTERN \".svn\" EXCLUDE\n# )\n\n## Mark other files for installation (e.g. launch and bag files, etc.)\n# install(FILES\n# # myfile1\n# # myfile2\n# DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION}\n# )\n\n#############\n## Testing ##\n#############\n\n## Add gtest based cpp test target and link libraries\n# catkin_add_gtest(${PROJECT_NAME}-test test/test_jpda_rospack.cpp)\n# if(TARGET ${PROJECT_NAME}-test)\n# target_link_libraries(${PROJECT_NAME}-test ${PROJECT_NAME})\n# endif()\n\n## Add folders to be run by python nosetests\n# catkin_add_nosetests(test)\n" }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2020 Stanford Vision and Learning Group\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "README.md", "content": "# JRMOT ROS package\n\nThe repository contains the code for the work \"JRMOT: A Real-Time 3D Multi-Object Tracker and a New Large-Scale Dataset\".\n\nNote that due to the global pandemic, this repository is still a work in progress. Updates will be made as soon as possible.\n\n## Introduction\n\nJRMOT is a 3D multi object tracking system that:\n- Is real-time\n- Is online\n- Fuses 2D and 3D information\n- Achieves State of the Art performance on KITTI\n\nWe also release JRDB:\n- A dataset with over 2 million annotated boxes and 3500 time consistent trajectories in 2D and 3D\n- Captured in social, human-centric settings\n- Captured by our social mobile-manipulator JackRabbot\n- Contains 360 degree cylindrical images, stereo camera images, 3D pointclouds and more sensing modalties\n\nAll information, including download links for JRDB can be found [here](https://jrdb.stanford.edu).\n\n## JRMOT\n![system overview](https://github.com/StanfordVL/JRMOT_ROS/blob/master/assets/framework.png)\n\n- Our system is built on top of state of the art 2D and 3D detectors (mask-RCNN and F-PointNet respectively). These detections are associated with predicted track locations at every time step. \n- Association is done via a novel feature fusion, as well as a cost selection procedure, followed by Kalman state gating and JPDA. \n- Given the JPDA output, we use both 2D and 3D detections in a novel multi-modal Kalman filter to update the track locations.\n\n\n## Using the code\n\nThere are 3 nodes forming parts of the ROS package:\n+ 3d_detector.py: Runs F-PointNet, which performs 3D detection and 3D feature extraction\n+ template.py: Runs Aligned-Re-ID, which performs 2D feature extraction\n+ tracker_3d_node.py: Performs tracking while taking both 2D detections + features and 3D detections + features as input\n\nThe launch file in the folder \"launch\" launches all 3 nodes.\n\n## Dependencies\n\nThe following are dependencies of the code:\n\n+ 2D detector: The 2D detector is not included in this package. To interface with your own 2D detector, please modify the file template.py to subscribe to the correct topic, and also to handle the conversion from ROS message to numpy array.\n+ Spencer People Tracking messages: The final tracker output is in a Spencer People Tracking message. Please install this package and include these message types.\n+ Various python packages: These can be found in [requirements.txt](./requirements.txt).. Please install all dependencies prior to running the code (including CUDA and cuDNN. Additionally, this code requires a solver called Gurobi. Instructions to install gurobipy can be found [here](https://www.gurobi.com/documentation/9.0/quickstart_mac/the_grb_python_interface_f.html).\n+ Weight files: The trained weights, (trained on JRDB) for FPointNet and Aligne-ReID can be found [here](https://drive.google.com/open?id=1YQinMPVWEI44KezS9inXe0mvVnm4aL3s).\n\n## Citation\n\nIf you find this work useful, please cite:\n```\n@INPROCEEDINGS{shenoi2020jrmot,\n author={A. {Shenoi} and M. {Patel} and J. {Gwak} and P. {Goebel} and A. {Sadeghian} and H. {Rezatofighi} and R. {Mart\\'in-Mart\\'in} and S. {Savarese}},\n booktitle={2020 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)}, \n title={JRMOT: A Real-Time 3D Multi-Object Tracker and a New Large-Scale Dataset}, \n year={2020},\n volume={},\n number={},\n pages={10335-10342},\n doi={10.1109/IROS45743.2020.9341635}}\n```\n\nIf you utilise our dataset, please also cite:\n\n```\n@article{martin2019jrdb,\n title={JRDB: A dataset and benchmark of egocentric visual perception for navigation in human environments},\n author={Mart{\\'i}n-Mart{\\'i}n, Roberto and Patel, Mihir and Rezatofighi, Hamid and Shenoi, Abhijeet and Gwak, JunYoung and Frankel, Eric and Sadeghian, Amir and Savarese, Silvio},\n journal={arXiv preprint arXiv:1910.11792},\n year={2019}\n}\n```\n## \n" }, { "path": "calib/cameras.yaml", "content": "stitching:\n radius: 3360000\n rotation: 0\n scalewidth: 1831\n crop: 1\ncameras:\n # camera order matters!\n sensor_0:\n width: 752\n height: 480\n D: -0.336591 0.159742 0.00012697 -7.22557e-05 -0.0461953\n # K = fx 0 cx\n # 0 fy cy\n # 0 0 1\n K: >\n 476.71 0 350.738\n 0 479.505 209.532\n 0 0 1\n R: >\n 0.999994 0.000654539 0.00340293\n -0.000654519 1 -6.81963e-06\n -0.00340293 4.59231e-06 0.999994\n T: -0.0104242 -3.70974 -56.9177\n sensor_1:\n width: 752\n height: 480\n D: -0.335073 0.151959 -0.000232061 0.00032014 -0.0396825\n K: >\n 483.254 0 365.33\n 0 485.78 210.953\n 0 0 1\n R: >\n 0.305706 -0.00895443 -0.952084\n 0.0110396 0.999922 -0.00585963\n 0.952062 -0.0087193 0.305781\n T: 0.93957 -4.05131 -52.03\n sensor_2:\n width: 752\n height: 480\n D: -0.338469 0.156256 -0.000385467 0.000295485 -0.0401965\n K: >\n 483.911 0 355.144\n 0 486.466 223.026\n 0 0 1\n R: >\n -0.806828 0.0136361 -0.590629\n 0.00870468 0.999899 0.011194\n 0.590723 0.00389039 -0.806865\n T: -0.25753 -6.54978 -47.7311\n sensor_3:\n width: 752\n height: 480\n D: -0.330848 0.14747 8.59247e-05 0.000262599 -0.0385311\n K: >\n 475.807 0 339.53\n 0 478.371 188.481\n 0 0 1\n R: >\n -0.811334 0.0033829 0.584574\n 0.00046071 0.999987 -0.00514746\n -0.584583 -0.00390699 -0.811324\n T: 2.72207 -6.82928 -45.9778\n sensor_4:\n width: 752\n height: 480\n D: -0.34064 0.168338 0.000147292 0.000229372 -0.0516133\n K: >\n 485.046 0 368.864\n 0 488.185 208.215\n 0 0 1\n R: >\n 0.310275 0.00160497 0.950645\n -0.00648686 0.999979 0.000428942\n -0.950625 -0.00629979 0.310279\n T: -0.333857 -5.12974 -56.0573\n sensor_5:\n width: 752\n height: 480\n D: -0.338422 0.163703 -0.000376267 7.73351e-06 -0.0479871\n K: >\n 478.406 0 353.499\n 0 481.322 190.225\n 0 0 1\n R: >\n 0.999995 0.00282205 0.00163291\n -0.00282345 0.999996 0.000852931\n -0.00163049 -0.000857537 0.999998\n T: -0.903588 -126.851 -56.6256\n sensor_6:\n width: 752\n height: 480\n D: -0.340676 0.165511 -0.00035978 0.000181532 -0.0493721\n K: >\n 480.459 0 362.503\n 0 482.924 197.949\n 0 0 1\n R: >\n 0.308288 -0.0110391 -0.951229\n -0.000933102 0.999929 -0.0119067\n 0.951293 0.00455829 0.308256\n T: 1.74525 -127.214 -51.7722\n sensor_7:\n width: 752\n height: 480\n D: -0.344379 0.170343 -0.000137847 0.000141047 -0.0510536\n K: >\n 486.491 0 361.559\n 0 489.22 210.547\n 0 0 1\n R: >\n -0.808201 0.0313998 -0.588068\n 0.026057 0.999506 0.0175574\n 0.588329 -0.00113337 -0.808621\n T: -2.56535 -129.191 -47.5803\n sensor_8:\n width: 752\n height: 480\n D: -0.331228 0.144696 0.000117553 0.000566449 -0.0343506\n K: >\n 476.708 0 354.16\n 0 479.424 209.383\n 0 0 1\n R: >\n -0.807384 -0.00296577 0.590019\n -0.0122001 0.999857 -0.0116688\n -0.589901 -0.0166195 -0.807305\n T: 3.39727 -129.381 -45.2409\n sensor_9:\n width: 752\n height: 480\n D: -0.345189 0.180808 0.000276465 0.000131868 -0.062103\n K: >\n 484.219 0 345.303\n 0 487.312 192.371\n 0 0 1\n R: >\n 0.308505 0.00370159 0.951215\n -0.00403535 0.999988 -0.00258261\n -0.951214 -0.00304174 0.308517\n T: 0.354966 -128.218 -54.0617\n\n" }, { "path": "calib/defaults.yaml", "content": "calibrated:\n # the lidar_to_rgb parameters allow tweaking of the transformation between lidar and rgb frames\n # the default transformation is taken from the TF Tree\n # NOTE: applied to the original (sensor/velodyne) frame [x forward, y left, z up]:\n lidar_upper_to_rgb:\n # in meters: [x,y,z]\n translation: [0, 0, -0.33529]\n # in radians: [x,y,z]\n rotation: [0, 0, 0.085]\n \n lidar_lower_to_rgb:\n\n translation: [0, 0, 0.13511]\n \n rotation: [0, 0, 0]\nimage:\n # all in pixels\n width: 3760\n height: 480\n # y-axis forward pixel offset (e.g. 3760/2 => 1880, b/c center of the cylindrical image is forward)\n # TODO: move into calibrated params, when auto-calibration is possible\n stitched_image_offset: 1880\n\nframes:\n # lookup for people transforms\n global: base_link\n # name of the rgb360 camera frame to which we wish to transform\n rgb360: occam\n" }, { "path": "config/featurepointnet.cfg", "content": "[general]\nnum_point = 1024\nmodel_path = /home/sibot/jr2_catkin_ws/src/jpda_rospack/src/fpointnet_jrdb/model.ckpt\n" }, { "path": "launch/jpda_tracker.launch", "content": "\n\n\n \n \n\n \n \n \n \n \n \n\n\n \n \n \n\n \n \n \n \n \n \n \n \n \n \n\n\n" }, { "path": "msg/__init__.py", "content": "" }, { "path": "msg/detection2d_with_feature.msg", "content": "# This message contains a 2D bounding box corresponding to the detection of a person\n# Also contains the feature of this person used for re-ID\n\nHeader header #header timestamp is time of frame acquisition\n\n\nuint64 x1 # x coordinate of the top left of the bounding box\nuint64 y1 # y coordinate of the top left of the bounding box\nuint64 x2 # x coordinate of the bottom right of the bounding box\nuint64 y2 # y coordinate of the bottom right of the bounding box\n\nfloat64[] feature # re-ID feature\n\nuint8 frame_det_id #unique id of this detection within this frame (used for associating 2D and 3D detections)\nbool valid # whether detection is valid (within the boundaries of the image and has minimum required size)" }, { "path": "msg/detection2d_with_feature_array.msg", "content": "Header header\ndetection2d_with_feature[] detection2d_with_features" }, { "path": "msg/detection3d_with_feature.msg", "content": "# This message contains a 3D bounding box corresponding to the detection of a person\n# Also contains the feature of this person used for re-ID\n\nHeader header #header timestamp is time of frame acquisition\n\n\nfloat32 x # x coordinate of the center of the bottom face of the bounding box\nfloat32 y # y coordinate of the center of the bottom face of the bounding box\nfloat32 z # x coordinate of the center of the bottom face of the bounding box\nfloat32 l # size of bounding box along x dimension\nfloat32 h # size of bounding box along y dimension\nfloat32 w # size of bounding box along z dimension\nfloat32 theta # rotation of bounding box with respect to the positive x axis\n\n\nfloat64[] feature # re-ID feature\n\nuint8 frame_det_id #unique id of this detection within this frame (used for associating 2D and 3D detections)\nbool valid # whether detection is valid (enough lidar points)" }, { "path": "msg/detection3d_with_feature_array.msg", "content": "Header header\ndetection3d_with_feature[] detection3d_with_features" }, { "path": "package.xml", "content": "\n\n jpda_rospack\n 0.0.1\n The jpda_rospack package\n\n \n \n \n ashenoi\n\n\n \n \n \n TODO\n\n\n \n \n \n \n\n\n \n \n \n \n\n\n \n \n \n \n \n \n \n \n \n message_generation\n \n \n \n \n \n message_runtime\n \n \n \n \n catkin\n roscpp\n rospy\n std_msgs\n vision_msgs\n roscpp\n rospy\n std_msgs\n vision_msgs\n roscpp\n rospy\n std_msgs\n vision_msgs\n\n\n \n \n \n\n \n\n" }, { "path": "paper_experiments/models/__init__.py", "content": "" }, { "path": "paper_experiments/models/aligned_reid_model.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.init as init\nimport torch.nn.functional as F\nimport torch.utils.model_zoo as model_zoo\nimport os\nimport math\n\n\nclass Model(nn.Module):\n def __init__(self, local_conv_out_channels=128, num_classes=None):\n super(Model, self).__init__()\n self.base = resnet50(pretrained=True)\n planes = 2048\n self.local_conv = nn.Conv2d(planes, local_conv_out_channels, 1)\n self.local_bn = nn.BatchNorm2d(local_conv_out_channels)\n self.local_relu = nn.ReLU(inplace=True)\n\n if num_classes is not None:\n self.fc = nn.Linear(planes, num_classes)\n init.normal(self.fc.weight, std=0.001)\n init.constant(self.fc.bias, 0)\n\n def forward(self, x):\n \"\"\"\n Returns:\n global_feat: shape [N, C]\n local_feat: shape [N, H, c]\n \"\"\"\n # shape [N, C, H, W]\n feat = self.base(x)\n global_feat = F.avg_pool2d(feat, feat.size()[2:])\n # shape [N, C]\n global_feat = global_feat.view(global_feat.size(0), -1)\n # shape [N, C, H, 1]\n local_feat = torch.mean(feat, -1, keepdim=True)\n local_feat = self.local_relu(self.local_bn(self.local_conv(local_feat)))\n # shape [N, H, c]\n local_feat = local_feat.squeeze(-1).permute(0, 2, 1)\n\n if hasattr(self, 'fc'):\n logits = self.fc(global_feat)\n return global_feat, local_feat, logits\n\n return global_feat, local_feat\n\n\n__all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101',\n 'resnet152']\n\nmodel_urls = {\n 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',\n 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',\n 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',\n 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',\n 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',\n}\nos.environ[\"TORCH_HOME\"] = \"./ResNet_Model\"\n\ndef conv3x3(in_planes, out_planes, stride=1):\n \"\"\"3x3 convolution with padding\"\"\"\n return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,\n padding=1, bias=False)\n\n\nclass BasicBlock(nn.Module):\n expansion = 1\n\n def __init__(self, inplanes, planes, stride=1, downsample=None):\n super(BasicBlock, self).__init__()\n self.conv1 = conv3x3(inplanes, planes, stride)\n self.bn1 = nn.BatchNorm2d(planes)\n self.relu = nn.ReLU(inplace=True)\n self.conv2 = conv3x3(planes, planes)\n self.bn2 = nn.BatchNorm2d(planes)\n self.downsample = downsample\n self.stride = stride\n\n def forward(self, x):\n residual = x\n\n out = self.conv1(x)\n out = self.bn1(out)\n out = self.relu(out)\n\n out = self.conv2(out)\n out = self.bn2(out)\n\n if self.downsample is not None:\n residual = self.downsample(x)\n\n out += residual\n out = self.relu(out)\n\n return out\n\n\nclass Bottleneck(nn.Module):\n expansion = 4\n\n def __init__(self, inplanes, planes, stride=1, downsample=None):\n super(Bottleneck, self).__init__()\n self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)\n self.bn1 = nn.BatchNorm2d(planes)\n self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,\n padding=1, bias=False)\n self.bn2 = nn.BatchNorm2d(planes)\n self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)\n self.bn3 = nn.BatchNorm2d(planes * 4)\n self.relu = nn.ReLU(inplace=True)\n self.downsample = downsample\n self.stride = stride\n\n def forward(self, x):\n residual = x\n\n out = self.conv1(x)\n out = self.bn1(out)\n out = self.relu(out)\n\n out = self.conv2(out)\n out = self.bn2(out)\n out = self.relu(out)\n\n out = self.conv3(out)\n out = self.bn3(out)\n\n if self.downsample is not None:\n residual = self.downsample(x)\n\n out += residual\n out = self.relu(out)\n\n return out\n\n\nclass ResNet(nn.Module):\n\n def __init__(self, block, layers):\n self.inplanes = 64\n super(ResNet, self).__init__()\n self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,\n bias=False)\n self.bn1 = nn.BatchNorm2d(64)\n self.relu = nn.ReLU(inplace=True)\n self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n self.layer1 = self._make_layer(block, 64, layers[0])\n self.layer2 = self._make_layer(block, 128, layers[1], stride=2)\n self.layer3 = self._make_layer(block, 256, layers[2], stride=2)\n self.layer4 = self._make_layer(block, 512, layers[3], stride=2)\n\n for m in self.modules():\n if isinstance(m, nn.Conv2d):\n n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n m.weight.data.normal_(0, math.sqrt(2. / n))\n elif isinstance(m, nn.BatchNorm2d):\n m.weight.data.fill_(1)\n m.bias.data.zero_()\n\n def _make_layer(self, block, planes, blocks, stride=1):\n downsample = None\n if stride != 1 or self.inplanes != planes * block.expansion:\n downsample = nn.Sequential(\n nn.Conv2d(self.inplanes, planes * block.expansion,\n kernel_size=1, stride=stride, bias=False),\n nn.BatchNorm2d(planes * block.expansion),\n )\n\n layers = []\n layers.append(block(self.inplanes, planes, stride, downsample))\n self.inplanes = planes * block.expansion\n for i in range(1, blocks):\n layers.append(block(self.inplanes, planes))\n\n return nn.Sequential(*layers)\n\n def forward(self, x):\n x = self.conv1(x)\n x = self.bn1(x)\n x = self.relu(x)\n x = self.maxpool(x)\n\n x = self.layer1(x)\n x = self.layer2(x)\n x = self.layer3(x)\n x = self.layer4(x)\n\n return x\n\n\ndef remove_fc(state_dict):\n \"\"\"Remove the fc layer parameters from state_dict.\"\"\"\n new_state_dict = state_dict.copy()\n for key, value in state_dict.items():\n if key.startswith('fc.'):\n del new_state_dict[key]\n return new_state_dict\n\n\ndef resnet18(pretrained=False):\n \"\"\"Constructs a ResNet-18 model.\n\n Args:\n pretrained (bool): If True, returns a model pre-trained on ImageNet\n \"\"\"\n model = ResNet(BasicBlock, [2, 2, 2, 2])\n if pretrained:\n model.load_state_dict(remove_fc(model_zoo.load_url(model_urls['resnet18'])))\n return model\n\n\ndef resnet34(pretrained=False):\n \"\"\"Constructs a ResNet-34 model.\n\n Args:\n pretrained (bool): If True, returns a model pre-trained on ImageNet\n \"\"\"\n model = ResNet(BasicBlock, [3, 4, 6, 3])\n if pretrained:\n model.load_state_dict(remove_fc(model_zoo.load_url(model_urls['resnet34'])))\n return model\n\n\ndef resnet50(pretrained=False):\n \"\"\"Constructs a ResNet-50 model.\n\n Args:\n pretrained (bool): If True, returns a model pre-trained on ImageNet\n \"\"\"\n model = ResNet(Bottleneck, [3, 4, 6, 3])\n if pretrained:\n model.load_state_dict(remove_fc(model_zoo.load_url(model_urls['resnet50'], model_dir=\"./ResNet_Model\")))### ADDED MODEL_DIR\n return model\n\n\ndef resnet101(pretrained=False):\n \"\"\"Constructs a ResNet-101 model.\n\n Args:\n pretrained (bool): If True, returns a model pre-trained on ImageNet\n \"\"\"\n model = ResNet(Bottleneck, [3, 4, 23, 3])\n if pretrained:\n model.load_state_dict(\n remove_fc(model_zoo.load_url(model_urls['resnet101'])))\n return model\n\n\ndef resnet152(pretrained=False):\n \"\"\"Constructs a ResNet-152 model.\n\n Args:\n pretrained (bool): If True, returns a model pre-trained on ImageNet\n \"\"\"\n model = ResNet(Bottleneck, [3, 8, 36, 3])\n if pretrained:\n model.load_state_dict(\n remove_fc(model_zoo.load_url(model_urls['resnet152'])))\n return model\n" }, { "path": "paper_experiments/models/combination_model.py", "content": "import pdb\n\nimport numpy as np\nimport torch.nn as nn\n\nclass CombiNet(nn.Module):\n\tdef __init__(self, in_dim = 2560, hidden_units = 512, out_dim = 2560):\n\t\tsuper().__init__()\n\t\tself.fc1 = nn.Linear(in_dim, 2*hidden_units)\n\t\t# self.bn1 = nn.BatchNorm1d(hidden_units)\n\t\tself.fc2 = nn.Linear(2*hidden_units, 2*hidden_units)\n\t\t# self.bn2 = nn.BatchNorm1d(2*hidden_units)\n\t\tself.fc3 = nn.Linear(2*hidden_units, out_dim)\n\t\tself.relu = nn.ReLU()\n\t\tself.apply(weight_init)\n\tdef forward(self, x):\n\t\t# out = nn.functional.normalize(x)\n\t\tskip = x\n\t\tout = self.fc1(x)\n\t\t# out = self.bn1(out)\n\t\tout = self.relu(out)\n\t\tout = self.fc2(out)\n\t\t# out = self.bn2(out)\n\t\tout = self.relu(out)\n\t\tout = self.fc3(out)\n\t\t# out = nn.functional.normalize(out)\n\t\tout += skip\n\t\treturn out\n\nclass CombiLSTM(nn.Module):\n\tdef __init__(self, in_dim = 2560, hidden_units = 512, out_dim = 2560):\n\t\tsuper().__init__()\n\t\tself.in_linear1 = nn.Linear(in_dim, hidden_units)\n\t\t# self.bn1 = nn.BatchNorm1d(hidden_units)\n\t\tself.in_linear2 = nn.Linear(hidden_units, hidden_units)\n\t\tself.rnn = nn.LSTM(input_size = hidden_units, hidden_size = hidden_units, dropout = 0)\n\t\tself.out_linear1 = nn.Linear(hidden_units, hidden_units)\n\t\t# self.bn2 = nn.BatchNorm1d(hidden_units)\n\t\tself.out_linear2 = nn.Linear(hidden_units, out_dim)\n\t\tself.relu = nn.ReLU()\n\t\tself.apply(weight_init)\n\n\tdef forward(self, x, hidden = None):\n\t\tout = nn.functional.normalize(x)\n\t\tskip = out\n\t\tout = self.in_linear1(out)\n\t\t# out = self.bn1(out)\n\t\tout = self.relu(out)\n\t\tout = self.in_linear2(out)\n\t\tout = out.unsqueeze(1) #Adding batch dimension\n\t\tif hidden is None:\n\t\t\tout, hidden = self.rnn(out)\n\t\telse:\n\t\t\tout, hidden = self.rnn(out, hidden)\n\n\t\tout = out.squeeze(1) #removing batch dimension\n\t\tout = self.out_linear1(out)\n\t\t# out = self.bn2(out)\n\t\tout = self.relu(out)\n\t\tout = self.out_linear2(out)\n\t\tout = nn.functional.normalize(out)\n\t\tout += skip\n\t\treturn out, hidden\n\ndef weight_init(m):\n\tif type(m)==nn.Linear:\n\t\tnn.init.xavier_normal_(m.weight, gain=np.sqrt(2))\n\telif type(m)==nn.LSTM:\n\t\tnn.init.xavier_normal_(m.weight_ih_l0)\n\t\tnn.init.xavier_normal_(m.weight_hh_l0)\n" }, { "path": "paper_experiments/models/deep_sort_model.py", "content": "import tensorflow as tf\nfrom skimage.transform import resize\nimport numpy as np\n\nclass ImageEncoder(object):\n\n def __init__(self, checkpoint_filename=\"weights/deep_sort_weights.pb\", input_name=\"images\",\n output_name=\"features\"):\n config = tf.ConfigProto()\n config.gpu_options.allow_growth = True\n self.session = tf.Session(config=config)\n with tf.gfile.GFile(checkpoint_filename, \"rb\") as file_handle:\n graph_def = tf.GraphDef()\n graph_def.ParseFromString(file_handle.read())\n tf.import_graph_def(graph_def, name=\"net\")\n self.input_var = tf.get_default_graph().get_tensor_by_name(\n \"net/%s:0\" % input_name)\n self.output_var = tf.get_default_graph().get_tensor_by_name(\n \"net/%s:0\" % output_name)\n\n assert len(self.output_var.get_shape()) == 2\n assert len(self.input_var.get_shape()) == 4\n self.feature_dim = self.output_var.get_shape().as_list()[-1]\n self.image_shape = self.input_var.get_shape().as_list()[1:]\n\n def __call__(self, data_x):\n #Resize input to expected size for model\n data_x = resize(data_x[0], self.image_shape, anti_aliasing=True, mode='reflect')\n data_x = np.expand_dims(data_x, 0)\n out = self.session.run(self.output_var, feed_dict={self.input_var: data_x})\n return out\n\nif __name__ == '__main__':\n encoder = ImageEncoder()" }, { "path": "paper_experiments/models/featurepointnet_model.py", "content": "import os, pdb\nimport numpy as np\nimport tensorflow as tf\ntf.logging.set_verbosity(tf.logging.ERROR)\nimport configparser\n\nimport utils.featurepointnet_tf_util as tf_util\nimport utils.featurepointnet_model_util as model_util\nfrom utils.calibration import Calibration, OmniCalibration\n\nbatch_size = 45 #TODO: Update if needed?\n\nclass FPointNet():\n def __init__(self, config_path):\n parser = configparser.SafeConfigParser()\n parser.read(config_path)\n self.num_point = parser.getint('general', 'num_point')\n self.model_path = parser.get('general', 'model_path')\n\n with tf.device('/gpu:'+str('0')):\n pointclouds_pl, one_hot_vec_pl, labels_pl, centers_pl, \\\n heading_class_label_pl, heading_residual_label_pl, \\\n size_class_label_pl, size_residual_label_pl = model_util.placeholder_inputs(batch_size, self.num_point)\n is_training_pl = tf.placeholder(tf.bool, shape=())\n end_points, depth_feature = self.get_model(pointclouds_pl, one_hot_vec_pl, is_training_pl)\n self.object_pointcloud = tf.placeholder(tf.float32, shape=(None, None, 3))\n #depth_feature = self.get_depth_feature_op(is_training_pl)\n loss = model_util.get_loss(labels_pl, centers_pl, heading_class_label_pl, heading_residual_label_pl, size_class_label_pl, size_residual_label_pl, end_points)\n self.saver = tf.train.Saver()\n\n # Create a session\n config = tf.ConfigProto()\n config.gpu_options.allow_growth = True\n config.allow_soft_placement = True\n self.sess = tf.Session(config=config)\n\n #Initialize variables\n self.sess.run(tf.global_variables_initializer())\n # Restore variables from disk.\n self.saver.restore(self.sess, self.model_path)\n self.ops = {'pointclouds_pl': pointclouds_pl,\n 'one_hot_vec_pl': one_hot_vec_pl,\n 'labels_pl': labels_pl,\n 'centers_pl': centers_pl,\n 'heading_class_label_pl': heading_class_label_pl,\n 'heading_residual_label_pl': heading_residual_label_pl,\n 'size_class_label_pl': size_class_label_pl,\n 'size_residual_label_pl': size_residual_label_pl,\n 'is_training_pl': is_training_pl,\n 'logits': end_points['mask_logits'],\n 'center': end_points['center'],\n 'end_points': end_points,\n 'depth_feature':depth_feature,\n 'loss': loss}\n\n # @profile\n def __call__(self, input_point_cloud, rot_angle, peds=False):\n '''\n one_hot_vec = np.zeros((batch_size, 3))\n feed_dict = {self.pointclouds_pl: input_point_cloud,\n self.one_hot_vec_pl: one_hot_vec,\n self.is_training_pl: False}\n features = self.sess.run(self.feature,feed_dict=feed_dict)\n return features '''\n\n ''' Run inference for frustum pointnets in batch mode '''\n \n one_hot_vec = np.zeros((batch_size,3))\n if peds:\n one_hot_vec[:, 1] = 1\n num_batches = input_point_cloud.shape[0]//batch_size + 1\n num_inputs = input_point_cloud.shape[0]\n if input_point_cloud.shape[0]%batch_size !=0:\n input_point_cloud = np.vstack([input_point_cloud, np.zeros((batch_size - input_point_cloud.shape[0]%batch_size, self.num_point, 4))])\n else:\n num_batches -= 1\n logits = np.zeros((input_point_cloud.shape[0], input_point_cloud.shape[1], 2))\n centers = np.zeros((input_point_cloud.shape[0], 3))\n heading_logits = np.zeros((input_point_cloud.shape[0], model_util.NUM_HEADING_BIN))\n heading_residuals = np.zeros((input_point_cloud.shape[0], model_util.NUM_HEADING_BIN))\n size_logits = np.zeros((input_point_cloud.shape[0], model_util.NUM_SIZE_CLUSTER))\n size_residuals = np.zeros((input_point_cloud.shape[0], model_util.NUM_SIZE_CLUSTER, 3))\n mask_mean_prob = np.zeros((input_point_cloud.shape[0],)) # Step scores\n heading_prob = np.zeros((input_point_cloud.shape[0],))\n size_prob = np.zeros((input_point_cloud.shape[0],))\n scores = np.zeros((input_point_cloud.shape[0],)) # 3D box score \n features = np.zeros((input_point_cloud.shape[0], 512))\n \n for i in range(num_batches): \n ep = self.ops['end_points'] \n feed_dict = {\\\n self.ops['pointclouds_pl']: input_point_cloud[i*batch_size: (i+1)*batch_size],\n self.ops['one_hot_vec_pl']: one_hot_vec,\n self.ops['is_training_pl']: False}\n\n batch_logits, batch_centers, \\\n batch_heading_scores, batch_heading_residuals, \\\n batch_size_scores, batch_size_residuals, batch_features = \\\n self.sess.run([self.ops['logits'], self.ops['center'],\n ep['heading_scores'], ep['heading_residuals'],\n ep['size_scores'], ep['size_residuals'], self.ops['depth_feature']],\n feed_dict=feed_dict)\n\n logits[i*batch_size: (i+1)*batch_size] = batch_logits\n centers[i*batch_size: (i+1)*batch_size] = batch_centers\n heading_logits[i*batch_size: (i+1)*batch_size] = batch_heading_scores\n heading_residuals[i*batch_size: (i+1)*batch_size] = batch_heading_residuals\n size_logits[i*batch_size: (i+1)*batch_size] = batch_size_scores\n size_residuals[i*batch_size: (i+1)*batch_size] = batch_size_residuals\n features[i*batch_size: (i+1)*batch_size] = batch_features[:,0,:]\n\n heading_cls = np.argmax(heading_logits, 1) # B\n size_cls = np.argmax(size_logits, 1) # B\n heading_res = np.vstack([heading_residuals[i, heading_cls[i]] for i in range(heading_cls.shape[0])])\n size_res = np.vstack([size_residuals[i, size_cls[i], :] for i in range(size_cls.shape[0])])\n\n #TODO: Make this accept batches if wanted\n boxes = []\n for i in range(num_inputs):\n box = np.array(model_util.from_prediction_to_label_format(centers[i], heading_cls[i], heading_res[i], size_cls[i], size_res[i], rot_angle[i]))\n box[6] = np.squeeze(box[6])\n swp = box[5]\n box[5] = box[4]\n box[4] = swp\n boxes.append(box) \n boxes = np.vstack(boxes)\n return boxes, mask_mean_prob[:num_inputs], features[:num_inputs]\n\n\n def get_instance_seg_v1_net(self, point_cloud, one_hot_vec, is_training, bn_decay, end_points):\n ''' 3D instance segmentation PointNet v1 network.\n Input:\n point_cloud: TF tensor in shape (B,N,4)\n frustum point clouds with XYZ and intensity in point channels\n XYZs are in frustum coordinate\n one_hot_vec: TF tensor in shape (B,3)\n length-3 vectors indicating predicted object type\n is_training: TF boolean scalar\n bn_decay: TF float scalar\n end_points: dict\n Output:\n logits: TF tensor in shape (B,N,2), scores for bkg/clutter and object\n end_points: dict\n '''\n num_point = point_cloud.get_shape()[1].value\n\n net = tf.expand_dims(point_cloud, 2)\n\n net = tf_util.conv2d(net, 64, [1,1],\n padding='VALID', stride=[1,1],\n bn=True, is_training=is_training,\n scope='conv1', bn_decay=bn_decay)\n net = tf_util.conv2d(net, 64, [1,1],\n padding='VALID', stride=[1,1],\n bn=True, is_training=is_training,\n scope='conv2', bn_decay=bn_decay)\n point_feat = tf_util.conv2d(net, 64, [1,1],\n padding='VALID', stride=[1,1],\n bn=True, is_training=is_training,\n scope='conv3', bn_decay=bn_decay)\n net = tf_util.conv2d(point_feat, 128, [1,1],\n padding='VALID', stride=[1,1],\n bn=True, is_training=is_training,\n scope='conv4', bn_decay=bn_decay)\n net = tf_util.conv2d(net, 1024, [1,1],\n padding='VALID', stride=[1,1],\n bn=True, is_training=is_training,\n scope='conv5', bn_decay=bn_decay)\n\n global_feat = tf_util.max_pool2d(net, [num_point,1],\n padding='VALID', scope='maxpool')\n\n global_feat = tf.concat([global_feat, tf.expand_dims(tf.expand_dims(one_hot_vec, 1), 1)], axis=3)\n global_feat_expand = tf.tile(global_feat, [1, num_point, 1, 1])\n concat_feat = tf.concat(axis=3, values=[point_feat, global_feat_expand])\n\n net = tf_util.conv2d(concat_feat, 512, [1,1],\n padding='VALID', stride=[1,1],\n bn=True, is_training=is_training,\n scope='conv6', bn_decay=bn_decay)\n net = tf_util.conv2d(net, 256, [1,1],\n padding='VALID', stride=[1,1],\n bn=True, is_training=is_training,\n scope='conv7', bn_decay=bn_decay)\n net = tf_util.conv2d(net, 128, [1,1],\n padding='VALID', stride=[1,1],\n bn=True, is_training=is_training,\n scope='conv8', bn_decay=bn_decay)\n net = tf_util.conv2d(net, 128, [1,1],\n padding='VALID', stride=[1,1],\n bn=True, is_training=is_training,\n scope='conv9', bn_decay=bn_decay)\n net = tf_util.dropout(net, is_training, 'dp1', keep_prob=0.5)\n\n logits = tf_util.conv2d(net, 2, [1,1],\n padding='VALID', stride=[1,1], activation_fn=None,\n scope='conv10')\n logits = tf.squeeze(logits, [2]) # BxNxC\n return logits, end_points\n \n def get_3d_box_estimation_v1_net(self, object_point_cloud, one_hot_vec,is_training, bn_decay, end_points):\n ''' 3D Box Estimation PointNet v1 network.\n Input:\n object_point_cloud: TF tensor in shape (B,M,C)\n point clouds in object coordinate\n one_hot_vec: TF tensor in shape (B,3)\n length-3 vectors indicating predicted object type\n Output:\n output: TF tensor in shape (B,3+NUM_HEADING_BIN*2+NUM_SIZE_CLUSTER*4)\n including box centers, heading bin class scores and residuals,\n and size cluster scores and residuals\n ''' \n num_point = object_point_cloud.get_shape()[1].value\n net = tf.expand_dims(object_point_cloud, 2)\n net = tf_util.conv2d(net, 128, [1,1],\n padding='VALID', stride=[1,1],\n bn=True, is_training=is_training,\n scope='conv-reg1', bn_decay=bn_decay)\n net = tf_util.conv2d(net, 128, [1,1],\n padding='VALID', stride=[1,1],\n bn=True, is_training=is_training,\n scope='conv-reg2', bn_decay=bn_decay)\n net = tf_util.conv2d(net, 256, [1,1],\n padding='VALID', stride=[1,1],\n bn=True, is_training=is_training,\n scope='conv-reg3', bn_decay=bn_decay)\n net = tf_util.conv2d(net, 512, [1,1],\n padding='VALID', stride=[1,1],\n bn=True, is_training=is_training,\n scope='conv-reg4', bn_decay=bn_decay)\n\n features = tf.reduce_max(net, axis = 1)\n\n net = tf_util.max_pool2d(net, [num_point,1],\n padding='VALID', scope='maxpool2')\n net = tf.squeeze(net, axis=[1,2])\n net = tf.concat([net, one_hot_vec], axis=1)\n net = tf_util.fully_connected(net, 512, scope='fc1', bn=True,\n is_training=is_training, bn_decay=bn_decay)\n net = tf_util.fully_connected(net, 256, scope='fc2', bn=True,\n is_training=is_training, bn_decay=bn_decay)\n\n # The first 3 numbers: box center coordinates (cx,cy,cz),\n # the next NUM_HEADING_BIN*2: heading bin class scores and bin residuals\n # next NUM_SIZE_CLUSTER*4: box cluster scores and residuals\n output = tf_util.fully_connected(net,\n 3+model_util.NUM_HEADING_BIN*2+model_util.NUM_SIZE_CLUSTER*4, activation_fn=None, scope='fc3')\n return output, end_points, features\n\n def get_model(self, point_cloud, one_hot_vec, is_training, bn_decay=None):\n ''' Frustum PointNets model. The model predict 3D object masks and\n amodel bounding boxes for objects in frustum point clouds.\n Input:\n point_cloud: TF tensor in shape (B,N,4)\n frustum point clouds with XYZ and intensity in point channels\n XYZs are in frustum coordinate\n one_hot_vec: TF tensor in shape (B,3)\n length-3 vectors indicating predicted object type\n is_training: TF boolean scalar\n bn_decay: TF float scalar\n Output:\n end_points: dict (map from name strings to TF tensors)\n '''\n end_points = {}\n \n # 3D Instance Segmentation PointNet\n logits, end_points = self.get_instance_seg_v1_net(\\\n point_cloud, one_hot_vec,\n is_training, bn_decay, end_points)\n end_points['mask_logits'] = logits\n\n # Masking\n # select masked points and translate to masked points' centroid\n object_point_cloud_xyz, mask_xyz_mean, end_points = \\\n model_util.point_cloud_masking(point_cloud, logits, end_points)\n\n # T-Net and coordinate translation\n center_delta, end_points = model_util.get_center_regression_net(\\\n object_point_cloud_xyz, one_hot_vec,\n is_training, bn_decay, end_points)\n stage1_center = center_delta + mask_xyz_mean # Bx3\n end_points['stage1_center'] = stage1_center\n # Get object point cloud in object coordinate\n object_point_cloud_xyz_new = \\\n object_point_cloud_xyz - tf.expand_dims(center_delta, 1)\n\n # Amodel Box Estimation PointNet\n output, end_points, features = self.get_3d_box_estimation_v1_net(\\\n object_point_cloud_xyz_new, one_hot_vec,\n is_training, bn_decay, end_points)\n\n # Parse output to 3D box parameters\n end_points = model_util.parse_output_to_tensors(output, end_points)\n end_points['center'] = end_points['center_boxnet'] + stage1_center # Bx3\n\n return end_points, features\n \n def get_depth_feature_op(self, is_training):\n\n net = tf.expand_dims(self.object_pointcloud, 2)\n net = tf_util.conv2d(net, 128, [1,1],\n padding='VALID', stride=[1,1],\n bn=True, is_training=is_training,\n scope='conv-reg1', bn_decay=None)\n net = tf_util.conv2d(net, 128, [1,1],\n padding='VALID', stride=[1,1],\n bn=True, is_training=is_training,\n scope='conv-reg2', bn_decay=None)\n net = tf_util.conv2d(net, 256, [1,1],\n padding='VALID', stride=[1,1],\n bn=True, is_training=is_training,\n scope='conv-reg3', bn_decay=None)\n net = tf_util.conv2d(net, 512, [1,1],\n padding='VALID', stride=[1,1],\n bn=True, is_training=is_training,\n scope='conv-reg4', bn_decay=None)\n net = tf.reduce_max(net, axis = 1)\n \n return net\n\n def get_depth_feature(self, object_pointcloud):\n \n feed_dict = {self.object_pointcloud:object_pointcloud, self.ops['is_training_pl']:False}\n depth_feature = self.sess.run([self.ops['depth_feature']], feed_dict = feed_dict)\n return depth_feature\n\n def softmax(self, x):\n ''' Numpy function for softmax'''\n shape = x.shape\n probs = np.exp(x - np.max(x, axis=len(shape)-1, keepdims=True))\n probs /= np.sum(probs, axis=len(shape)-1, keepdims=True)\n return probs\n\ndef create_depth_model(model, config_path):\n #Note that folder path must be the folder containing the config.yaml file if omni_camera is True\n if model == 'FPointNet':\n return FPointNet(config_path)\n elif model == 'PointNet':\n return PointNet(config_path)" }, { "path": "paper_experiments/models/pointnet_model.py", "content": "import os, pdb\nimport tensorflow as tf\ntf.logging.set_verbosity(tf.logging.ERROR)\nimport configparser\nfrom utils.pointnet_transform_nets import input_transform_net, feature_transform_net\nimport utils.pointnet_tf_util as pointnet_tf_util\n\n\nclass PointNet():\n def __init__(self, config_path):\n parser = configparser.SafeConfigParser()\n parser.read(config_path)\n num_points = parser.getint('general', 'num_point')\n depth_model_path = parser.get('general', 'depth_model_path')\n\n with tf.device('/gpu:'+str(0)):\n self.pointclouds_pl, _ = self.placeholder_inputs(1, num_points)\n self.is_training_pl = tf.placeholder(tf.bool, shape=())\n\n # simple model\n feature = self.get_model(self.pointclouds_pl, self.is_training_pl)\n self.feature = feature\n # Add ops to save and restore all the variables.\n \n self.saver = tf.train.Saver()\n #Create session\n config = tf.ConfigProto()\n config.gpu_options.allow_growth = True\n config.allow_soft_placement = True\n config.log_device_placement = False\n self.sess = tf.Session(config=config)\n #Initialize variables\n self.sess.run(tf.global_variables_initializer())\n #Restore model weights\n self.saver.restore(self.sess, depth_model_path)\n\n def __call__(self, input_point_cloud):\n feed_dict = {self.pointclouds_pl: input_point_cloud,\n self.is_training_pl: False}\n features = self.sess.run(self.feature,feed_dict=feed_dict)\n return features\n\n def placeholder_inputs(self, batch_size, num_point):\n pointclouds_pl = tf.placeholder(tf.float32, shape=(batch_size, None, 3))\n labels_pl = tf.placeholder(tf.int32, shape=(batch_size))\n return pointclouds_pl, labels_pl\n\n\n def get_model(self, point_cloud, is_training, bn_decay=None):\n \"\"\" Classification PointNet, input is BxNx3, output Bx40 \"\"\"\n batch_size = point_cloud.get_shape()[0].value\n end_points = {}\n\n with tf.variable_scope('transform_net1', reuse=tf.AUTO_REUSE) as sc:\n transform = input_transform_net(point_cloud, is_training, bn_decay, K=3)\n point_cloud_transformed = tf.matmul(point_cloud, transform)\n input_image = tf.expand_dims(point_cloud_transformed, -1)\n\n net = pointnet_tf_util.conv2d(input_image, 64, [1,3],\n padding='VALID', stride=[1,1],\n bn=True, is_training=is_training,\n scope='conv1', bn_decay=bn_decay)\n net = pointnet_tf_util.conv2d(net, 64, [1,1],\n padding='VALID', stride=[1,1],\n bn=True, is_training=is_training,\n scope='conv2', bn_decay=bn_decay)\n\n with tf.variable_scope('transform_net2', reuse=tf.AUTO_REUSE) as sc:\n transform = feature_transform_net(net, is_training, bn_decay, K=64)\n end_points['transform'] = transform\n net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform)\n net_transformed = tf.expand_dims(net_transformed, [2])\n\n net = pointnet_tf_util.conv2d(net_transformed, 64, [1,1],\n padding='VALID', stride=[1,1],\n bn=True, is_training=is_training,\n scope='conv3', bn_decay=bn_decay)\n net = pointnet_tf_util.conv2d(net, 128, [1,1],\n padding='VALID', stride=[1,1],\n bn=True, is_training=is_training,\n scope='conv4', bn_decay=bn_decay)\n net = pointnet_tf_util.conv2d(net, 1024, [1,1],\n padding='VALID', stride=[1,1],\n bn=True, is_training=is_training,\n scope='conv5', bn_decay=bn_decay)\n\n # Symmetric function: max pooling\n net = tf.reduce_max(net, axis = 1)\n\n net = tf.reshape(net, [batch_size, -1])\n feature = net\n\n return feature\n\n\n def get_loss(self, pred, label, end_points, reg_weight=0.001):\n \"\"\" pred: B*NUM_CLASSES,\n label: B, \"\"\"\n loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=pred, labels=label)\n classify_loss = tf.reduce_mean(loss)\n tf.summary.scalar('classify loss', classify_loss)\n\n # Enforce the transformation as orthogonal matrix\n transform = end_points['transform'] # BxKxK\n K = transform.get_shape()[1].value\n mat_diff = tf.matmul(transform, tf.transpose(transform, perm=[0,2,1]))\n mat_diff = mat_diff - tf.constant(np.eye(K), dtype=tf.float32)\n mat_diff_loss = tf.nn.l2_loss(mat_diff) \n tf.summary.scalar('mat loss', mat_diff_loss)\n\n return classify_loss + mat_diff_loss * reg_weight\n " }, { "path": "paper_experiments/models/resnet_reid_models.py", "content": "import torch\nimport torch.nn as nn\nfrom torch.autograd import Variable\nimport torchvision.models as models\nfrom torchvision import transforms\nimport torch.nn.functional as F\n\nclass FeatureResNet(nn.Module):\n def __init__(self,n_layers=50,pretrained=True):\n super(FeatureResNet,self).__init__()\n if n_layers == 50:\n old_model= models.resnet50(pretrained=pretrained)\n elif n_layers == 34:\n old_model= models.resnet34(pretrained=pretrained)\n elif n_layers == 18:\n old_model= models.resnet18(pretrained=pretrained)\n else:\n raise NotImplementedError('resnet%s is not found'%(n_layers))\n\n for name,modules in old_model._modules.items():\n if name.find('fc') == -1:\n self.add_module(name,modules)\n self.output_dim = old_model.fc.in_features\n self.pretrained = pretrained\n def forward(self,x):\n for name,module in self._modules.items():\n x = nn.parallel.data_parallel(module, x)\n return x.view(x.size(0), -1)\n\nclass ResNet(nn.Module):\n def __init__(self,n_id,n_layers=50,pretrained=True):\n super(ResNet,self).__init__()\n if n_layers == 50:\n old_model= models.resnet50(pretrained=pretrained)\n elif n_layers == 34:\n old_model= models.resnet34(pretrained=pretrained)\n elif n_layers == 18:\n old_model= models.resnet18(pretrained=pretrained)\n else:\n raise NotImplementedError('resnet%s is not found'%(n_layers))\n\n for name,modules in old_model._modules.items():\n self.add_module(name,modules)\n self.fc = nn.Linear(self.fc.in_features,n_id)\n #########\n self.pretrained = pretrained\n def forward(self,x):\n for name,module in self._modules.items():\n if name != 'fc':\n x = module(x)\n out = self.fc(x.view(x.size(0),-1))\n return out, x.view(x.size(0), -1)\n\nclass NLayersFC(nn.Module):\n def __init__(self, in_dim, out_dim, hidden_dim=1, n_layers=0):\n super(NLayersFC, self).__init__()\n if n_layers == 0:\n model = [nn.Linear(in_dim, out_dim)]\n else:\n model = []\n model += [nn.Linear(in_dim, hidden_dim),\n nn.ReLU(True)]\n for i in range(n_layers-1):\n model += [nn.Linear(hidden_dim, hidden_dim),\n nn.ReLU(True)]\n model += [nn.Linear(hidden_dim, out_dim)]\n self.model = nn.Sequential(*model)\n\n def forward(self, x):\n return self.model(x)\n\nclass ICT_ResNet(nn.Module):\n def __init__(self,n_id,n_color,n_type,n_layers=50,pretrained=True):\n super(ICT_ResNet,self).__init__()\n if n_layers == 50:\n old_model= models.resnet50(pretrained=pretrained)\n elif n_layers == 34:\n old_model= models.resnet34(pretrained=pretrained)\n elif n_layers == 18:\n old_model= models.resnet18(pretrained=pretrained)\n else:\n raise NotImplementedError('resnet%s is not found'%(n_layers))\n\n for name,modules in old_model._modules.items():\n self.add_module(name,modules)\n self.fc = nn.Linear(self.fc.in_features,n_id)\n self.fc_c = nn.Linear(self.fc.in_features,n_color)\n self.fc_t = nn.Linear(self.fc.in_features,n_type)\n #########\n self.pretrained = pretrained\n def forward(self,x):\n for name,module in self._modules.items():\n if name.find('fc')==-1:\n x = module(x)\n x = x.view(x.size(0),-1)\n x_i = self.fc(x)\n x_c = self.fc_c(x)\n x_t = self.fc_t(x)\n return x_i,x_c,x_t\n\nclass TripletNet(nn.Module):\n def __init__(self, net):\n super(TripletNet, self).__init__()\n self.net = net\n\n def forward(self, x, y, z):\n pred_x, feat_x = self.net(x)\n pred_y, feat_y = self.net(y)\n pred_z, feat_z = self.net(z)\n dist_pos = F.pairwise_distance(feat_x, feat_y, 2)\n dist_neg = F.pairwise_distance(feat_x, feat_z, 2)\n return dist_pos, dist_neg, pred_x, pred_y, pred_z\n\nif __name__ == '__main__':\n netM = ICT_ResNet(n_id=1000,n_color=7,n_type=7,n_layers=18,pretrained=True).cuda()\n\n print(netM)\n output = netM(Variable(torch.ones(1,3,224,224).cuda()/2.))\n print(output[0].size())\n print(output[1].size())\n print(output[2].size())\n" }, { "path": "paper_experiments/models/yolo_models.py", "content": "from __future__ import division\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\nimport numpy as np\n\nfrom PIL import Image\n\nfrom utils.yolo_utils.parse_config import *\nfrom utils.yolo_utils.utils import build_targets\nfrom collections import defaultdict\n\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\n\n\ndef create_modules(module_defs):\n \"\"\"\n Constructs module list of layer blocks from module configuration in module_defs\n \"\"\"\n hyperparams = module_defs.pop(0)\n output_filters = [int(hyperparams[\"channels\"])]\n module_list = nn.ModuleList()\n for i, module_def in enumerate(module_defs):\n modules = nn.Sequential()\n\n if module_def[\"type\"] == \"convolutional\":\n bn = int(module_def[\"batch_normalize\"])\n filters = int(module_def[\"filters\"])\n kernel_size = int(module_def[\"size\"])\n pad = (kernel_size - 1) // 2 if int(module_def[\"pad\"]) else 0\n modules.add_module(\n \"conv_%d\" % i,\n nn.Conv2d(\n in_channels=output_filters[-1],\n out_channels=filters,\n kernel_size=kernel_size,\n stride=int(module_def[\"stride\"]),\n padding=pad,\n bias=not bn,\n ),\n )\n if bn:\n modules.add_module(\"batch_norm_%d\" % i, nn.BatchNorm2d(filters))\n if module_def[\"activation\"] == \"leaky\":\n modules.add_module(\"leaky_%d\" % i, nn.LeakyReLU(0.1))\n\n elif module_def[\"type\"] == \"maxpool\":\n kernel_size = int(module_def[\"size\"])\n stride = int(module_def[\"stride\"])\n if kernel_size == 2 and stride == 1:\n padding = nn.ZeroPad2d((0, 1, 0, 1))\n modules.add_module(\"_debug_padding_%d\" % i, padding)\n maxpool = nn.MaxPool2d(\n kernel_size=int(module_def[\"size\"]),\n stride=int(module_def[\"stride\"]),\n padding=int((kernel_size - 1) // 2),\n )\n modules.add_module(\"maxpool_%d\" % i, maxpool)\n\n elif module_def[\"type\"] == \"upsample\":\n upsample = Interpolate(scale_factor=int(module_def[\"stride\"]), mode=\"nearest\")\n modules.add_module(\"upsample_%d\" % i, upsample)\n\n elif module_def[\"type\"] == \"route\":\n layers = [int(x) for x in module_def[\"layers\"].split(\",\")]\n filters = sum([output_filters[layer_i] for layer_i in layers])\n modules.add_module(\"route_%d\" % i, EmptyLayer())\n\n elif module_def[\"type\"] == \"shortcut\":\n filters = output_filters[int(module_def[\"from\"])]\n modules.add_module(\"shortcut_%d\" % i, EmptyLayer())\n\n elif module_def[\"type\"] == \"yolo\":\n anchor_idxs = [int(x) for x in module_def[\"mask\"].split(\",\")]\n # Extract anchors\n anchors = [int(x) for x in module_def[\"anchors\"].split(\",\")]\n anchors = [(anchors[i], anchors[i + 1]) for i in range(0, len(anchors), 2)]\n anchors = [anchors[i] for i in anchor_idxs]\n num_classes = int(module_def[\"classes\"])\n img_height = int(hyperparams[\"height\"])\n # Define detection layer\n yolo_layer = YOLOLayer(anchors, num_classes, img_height)\n modules.add_module(\"yolo_%d\" % i, yolo_layer)\n # Register module list and number of output filters\n module_list.append(modules)\n output_filters.append(filters)\n\n return hyperparams, module_list\n\n\nclass EmptyLayer(nn.Module):\n \"\"\"Placeholder for 'route' and 'shortcut' layers\"\"\"\n\n def __init__(self):\n super(EmptyLayer, self).__init__()\n\nclass Interpolate(nn.Module):\n def __init__(self, scale_factor, mode):\n super(Interpolate, self).__init__()\n self.interp = nn.functional.interpolate\n self.scale_factor = scale_factor\n self.mode = mode\n \n def forward(self, x):\n x = self.interp(x, scale_factor=self.scale_factor, mode=self.mode)\n return x\n\nclass YOLOLayer(nn.Module):\n \"\"\"Detection layer\"\"\"\n\n def __init__(self, anchors, num_classes, img_dim):\n super(YOLOLayer, self).__init__()\n self.anchors = anchors\n self.num_anchors = len(anchors)\n self.num_classes = num_classes\n self.bbox_attrs = 5 + num_classes\n self.image_dim = img_dim\n self.ignore_thres = 0.5\n self.lambda_coord = 1\n\n self.mse_loss = nn.MSELoss(reduction = 'elementwise_mean') # Coordinate loss\n self.bce_loss = nn.BCELoss(reduction = 'elementwise_mean') # Confidence loss\n self.ce_loss = nn.CrossEntropyLoss() # Class loss\n\n def forward(self, x, targets=None):\n nA = self.num_anchors\n nB = x.size(0)\n nG = x.size(2)\n stride = self.image_dim / nG\n\n # Tensors for cuda support\n FloatTensor = torch.cuda.FloatTensor if x.is_cuda else torch.FloatTensor\n LongTensor = torch.cuda.LongTensor if x.is_cuda else torch.LongTensor\n ByteTensor = torch.cuda.ByteTensor if x.is_cuda else torch.ByteTensor\n\n prediction = x.view(nB, nA, self.bbox_attrs, nG, nG).permute(0, 1, 3, 4, 2).contiguous()\n\n # Get outputs\n x = torch.sigmoid(prediction[..., 0]) # Center x\n y = torch.sigmoid(prediction[..., 1]) # Center y\n w = prediction[..., 2] # Width\n h = prediction[..., 3] # Height\n pred_conf = torch.sigmoid(prediction[..., 4]) # Conf\n pred_cls = torch.sigmoid(prediction[..., 5:]) # Cls pred.\n\n # Calculate offsets for each grid\n grid_x = torch.arange(nG).repeat(nG, 1).view([1, 1, nG, nG]).type(FloatTensor)\n grid_y = torch.arange(nG).repeat(nG, 1).t().view([1, 1, nG, nG]).type(FloatTensor)\n scaled_anchors = FloatTensor([(a_w / stride, a_h / stride) for a_w, a_h in self.anchors])\n anchor_w = scaled_anchors[:, 0:1].view((1, nA, 1, 1))\n anchor_h = scaled_anchors[:, 1:2].view((1, nA, 1, 1))\n\n # Add offset and scale with anchors\n pred_boxes = FloatTensor(prediction[..., :4].shape)\n pred_boxes[..., 0] = x.data + grid_x\n pred_boxes[..., 1] = y.data + grid_y\n pred_boxes[..., 2] = torch.exp(w.data) * anchor_w\n pred_boxes[..., 3] = torch.exp(h.data) * anchor_h\n\n # Training\n if targets is not None:\n\n if x.is_cuda:\n self.mse_loss = self.mse_loss.cuda()\n self.bce_loss = self.bce_loss.cuda()\n self.ce_loss = self.ce_loss.cuda()\n\n nGT, nCorrect, mask, conf_mask, tx, ty, tw, th, tconf, tcls = build_targets(\n pred_boxes=pred_boxes.cpu().data,\n pred_conf=pred_conf.cpu().data,\n pred_cls=pred_cls.cpu().data,\n target=targets.cpu().data,\n anchors=scaled_anchors.cpu().data,\n num_anchors=nA,\n num_classes=self.num_classes,\n grid_size=nG,\n ignore_thres=self.ignore_thres,\n img_dim=self.image_dim,\n )\n\n nProposals = int((pred_conf > 0.5).sum().item())\n recall = float(nCorrect / nGT) if nGT else 1\n precision = float(nCorrect / nProposals)\n\n # Handle masks\n mask = Variable(mask.type(ByteTensor))\n conf_mask = Variable(conf_mask.type(ByteTensor))\n\n # Handle target variables\n tx = Variable(tx.type(FloatTensor), requires_grad=False)\n ty = Variable(ty.type(FloatTensor), requires_grad=False)\n tw = Variable(tw.type(FloatTensor), requires_grad=False)\n th = Variable(th.type(FloatTensor), requires_grad=False)\n tconf = Variable(tconf.type(FloatTensor), requires_grad=False)\n tcls = Variable(tcls.type(LongTensor), requires_grad=False)\n\n # Get conf mask where gt and where there is no gt\n conf_mask_true = mask\n conf_mask_false = conf_mask - mask\n\n # Mask outputs to ignore non-existing objects\n loss_x = self.mse_loss(x[mask], tx[mask])\n loss_y = self.mse_loss(y[mask], ty[mask])\n loss_w = self.mse_loss(w[mask], tw[mask])\n loss_h = self.mse_loss(h[mask], th[mask])\n loss_conf = self.bce_loss(pred_conf[conf_mask_false], tconf[conf_mask_false]) + self.bce_loss(\n pred_conf[conf_mask_true], tconf[conf_mask_true]\n )\n loss_cls = (1 / nB) * self.ce_loss(pred_cls[mask], torch.argmax(tcls[mask], 1))\n loss = loss_x + loss_y + loss_w + loss_h + loss_conf + loss_cls\n\n return (\n loss,\n loss_x.item(),\n loss_y.item(),\n loss_w.item(),\n loss_h.item(),\n loss_conf.item(),\n loss_cls.item(),\n recall,\n precision,\n )\n\n else:\n # If not in training phase return predictions\n output = torch.cat(\n (\n pred_boxes.view(nB, -1, 4) * stride,\n pred_conf.view(nB, -1, 1),\n pred_cls.view(nB, -1, self.num_classes),\n ),\n -1,\n )\n return output\n\n\nclass Darknet(nn.Module):\n \"\"\"YOLOv3 object detection model\"\"\"\n\n def __init__(self, config_path):\n super(Darknet, self).__init__()\n self.module_defs = parse_model_config(config_path)\n self.hyperparams, self.module_list = create_modules(self.module_defs)\n self.seen = 0\n self.header_info = np.array([0, 0, 0, self.seen, 0])\n self.loss_names = [\"x\", \"y\", \"w\", \"h\", \"conf\", \"cls\", \"recall\", \"precision\"]\n self.load_weights(self.module_defs[-1]['path'])\n\n def forward(self, x, targets=None):\n is_training = targets is not None\n output = []\n self.losses = defaultdict(float)\n layer_outputs = []\n for i, (module_def, module) in enumerate(zip(self.module_defs, self.module_list)):\n if module_def[\"type\"] in [\"convolutional\", \"upsample\", \"maxpool\"]:\n x = module(x)\n elif module_def[\"type\"] == \"route\":\n layer_i = [int(x) for x in module_def[\"layers\"].split(\",\")]\n x = torch.cat([layer_outputs[i] for i in layer_i], 1)\n elif module_def[\"type\"] == \"shortcut\":\n layer_i = int(module_def[\"from\"])\n x = layer_outputs[-1] + layer_outputs[layer_i]\n elif module_def[\"type\"] == \"yolo\":\n # Train phase: get loss\n if is_training:\n x, *losses = module[0](x, targets)\n for name, loss in zip(self.loss_names, losses):\n self.losses[name] += loss\n # Test phase: Get detections\n else:\n x = module(x)\n output.append(x)\n layer_outputs.append(x)\n\n self.losses[\"recall\"] /= 3\n self.losses[\"precision\"] /= 3\n return sum(output) if is_training else torch.cat(output, 1)\n\n def load_weights(self, weights_path):\n \"\"\"Parses and loads the weights stored in 'weights_path'\"\"\"\n\n # Open the weights file\n fp = open(weights_path, \"rb\")\n header = np.fromfile(fp, dtype=np.int32, count=5) # First five are header values\n\n # Needed to write header when saving weights\n self.header_info = header\n\n self.seen = header[3]\n weights = np.fromfile(fp, dtype=np.float32) # The rest are weights\n fp.close()\n\n ptr = 0\n for i, (module_def, module) in enumerate(zip(self.module_defs, self.module_list)):\n if module_def[\"type\"] == \"convolutional\":\n conv_layer = module[0]\n if module_def[\"batch_normalize\"]:\n # Load BN bias, weights, running mean and running variance\n bn_layer = module[1]\n num_b = bn_layer.bias.numel() # Number of biases\n # Bias\n bn_b = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(bn_layer.bias)\n bn_layer.bias.data.copy_(bn_b)\n ptr += num_b\n # Weight\n bn_w = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(bn_layer.weight)\n bn_layer.weight.data.copy_(bn_w)\n ptr += num_b\n # Running Mean\n bn_rm = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(bn_layer.running_mean)\n bn_layer.running_mean.data.copy_(bn_rm)\n ptr += num_b\n # Running Var\n bn_rv = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(bn_layer.running_var)\n bn_layer.running_var.data.copy_(bn_rv)\n ptr += num_b\n else:\n # Load conv. bias\n num_b = conv_layer.bias.numel()\n conv_b = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(conv_layer.bias)\n conv_layer.bias.data.copy_(conv_b)\n ptr += num_b\n # Load conv. weights\n num_w = conv_layer.weight.numel()\n conv_w = torch.from_numpy(weights[ptr : ptr + num_w]).view_as(conv_layer.weight)\n conv_layer.weight.data.copy_(conv_w)\n ptr += num_w\n\n \"\"\"\n @:param path - path of the new weights file\n @:param cutoff - save layers between 0 and cutoff (cutoff = -1 -> all are saved)\n \"\"\"\n\n def save_weights(self, path, cutoff=-1):\n\n fp = open(path, \"wb\")\n self.header_info[3] = self.seen\n self.header_info.tofile(fp)\n\n # Iterate through layers\n for i, (module_def, module) in enumerate(zip(self.module_defs[:cutoff], self.module_list[:cutoff])):\n if module_def[\"type\"] == \"convolutional\":\n conv_layer = module[0]\n # If batch norm, load bn first\n if module_def[\"batch_normalize\"]:\n bn_layer = module[1]\n bn_layer.bias.data.cpu().numpy().tofile(fp)\n bn_layer.weight.data.cpu().numpy().tofile(fp)\n bn_layer.running_mean.data.cpu().numpy().tofile(fp)\n bn_layer.running_var.data.cpu().numpy().tofile(fp)\n # Load conv bias\n else:\n conv_layer.bias.data.cpu().numpy().tofile(fp)\n # Load conv weights\n conv_layer.weight.data.cpu().numpy().tofile(fp)\n\n fp.close()\n" }, { "path": "paper_experiments/requirements.txt", "content": "absl-py==0.7.0\nastor==0.7.1\nbackcall==0.1.0\nbleach==3.3.0\ncatkin-pkg==0.4.12\ncertifi==2018.11.29\nchardet==3.0.4\ncloudpickle==0.7.0\ncupy==6.0.0\ncycler==0.10.0\nCython==0.29.7\ndask==2021.10.0\ndecorator==4.3.2\ndefusedxml==0.5.0\ndocutils==0.14\nentrypoints==0.3\nfastrlock==0.4\nffmpeg==1.4\ngast==0.2.2\ngrpcio==1.18.0\ngurobipy==8.1.0\nhtml5lib==0.9999999\nidna==2.8\nimageio==2.5.0\nimageio-ffmpeg==0.3.0\nipydatawidgets==4.0.0\nipykernel==5.1.0\nipympl==0.2.1\nipython==7.16.3\nipython-genutils==0.2.0\nipyvolume==0.5.1\nipywebrtc==0.4.3\nipywidgets==7.4.2\njedi==0.13.2\nJinja2==2.11.3\njsonschema==2.6.0\njupyter-client==5.2.4\njupyter-core==4.4.0\njupyterlab==1.2.21\njupyterlab-server==0.2.0\nkiwisolver==1.0.1\nlap==0.4.0\nlapjv==1.3.1\nline-profiler==2.1.1\nllvmlite==0.28.0\nMarkdown==3.0.1\nMarkupSafe==1.1.0\nmatplotlib==3.0.2\nmistune==0.8.4\nnbconvert==5.4.1\nnbformat==4.4.0\nnetworkx==2.2\nnotebook==6.4.1\nnumba==0.43.1\nnumpy==1.21.0\nopen3d-python==0.7.0.0\nopencv-python==4.2.0.32\npandas==0.24.1\npandocfilters==1.4.2\nparso==0.3.3\npexpect==4.6.0\npickleshare==0.7.5\nPillow==9.0.0\npptk==0.1.0\nprometheus-client==0.5.0\nprompt-toolkit==2.0.8\nprotobuf==3.6.1\nptyprocess==0.6.0\npycocotools==2.0.0\nPygments==2.7.4\npyparsing==2.3.1\npypcd==0.1.1\npython-dateutil==2.8.0\npython-lzf==0.2.4\npythreejs==2.0.2\npytz==2018.9\nPyWavelets==1.0.1\nPyYAML==5.4\npyzmq==17.1.2\nrequests==2.21.0\nrospkg==1.1.9\nscikit-image==0.14.2\nscikit-learn==0.20.2\nscipy==1.2.0\nseaborn==0.9.0\nSend2Trash==1.5.0\nsix==1.12.0\nsklearn==0.0\ntensorboard==1.8.0\ntensorboardX==1.6\ntensorflow-gpu==2.5.2\ntermcolor==1.1.0\nterminado==0.8.1\ntestpath==0.4.2\ntoolz==0.9.0\ntorch==1.0.1\ntorchvision==0.2.1\ntornado==5.1.1\ntqdm==4.30.0\ntraitlets==4.3.2\ntraittypes==0.2.1\nurllib3==1.26.5\nwcwidth==0.1.7\nWerkzeug==0.15.3\nwidgetsnbextension==3.4.2\n" }, { "path": "paper_experiments/track.py", "content": "import open3d as o3d\nimport torch\nimport argparse\nimport os, pdb, sys, copy, pickle\nimport time\nimport random\nimport numpy as np\nimport tensorflow as tf\nfrom torch.utils.data import DataLoader\nfrom tqdm import tqdm\n\nfrom models.aligned_reid_model import Model as aligned_reid_model\nfrom utils.yolo_utils.utils import non_max_suppression, load_classes\nfrom models.combination_model import CombiNet, CombiLSTM\nfrom utils.dataset import SequenceDataset, STIPDataset, collate_fn\nfrom models.deep_sort_model import ImageEncoder as deep_sort_model\nfrom utils.tracker import Tracker\nfrom utils.tracker_3d import Tracker_3d\nfrom utils.deep_sort_utils import non_max_suppression as deepsort_nms\nfrom utils.visualise import draw_track\nfrom utils.read_detections import read_ground_truth_2d_detections, read_ground_truth_3d_detections\nfrom utils.tracking_utils import create_detector, convert_detections, combine_features\nfrom utils.tracking_utils import non_max_suppression_3D, non_max_suppression_3D_prime\nfrom utils.aligned_reid_utils import generate_features, generate_features_batched, get_image_patches, create_appearance_model\nfrom utils.featurepointnet_model_util import generate_detections_3d, convert_depth_features\nfrom models.featurepointnet_model import create_depth_model\n\ndef parse_arguments():\n parser = argparse.ArgumentParser()\n parser.add_argument('--sequence_folder', type=str, default='data/KITTI/sequences/0001', help='path to image sequence')\n parser.add_argument('--output_folder', type=str, default='results', help='output folder')\n parser.add_argument('--aligned_reid_ckpt', type=str, default='weights/aligned_reid_market_weights.ckpt', help='path to model config file')\n parser.add_argument('--resnet_reid_ckpt', type=str, default='weights/resnet_reid.ckpt', help='path to model config file')\n parser.add_argument('--depth_model', type=str, default='FPointNet', help='type of depth model to use')\n parser.add_argument('--depth_config_path', type=str, default='config/featurepointnet.cfg', help='path to model config file')\n parser.add_argument('--appearance_model', type=str, default='resnet_reid', help='type of appearance model to use aligned_reid or deepsort or resnet_reid')\n parser.add_argument('--conf_thres', type=float, default=0.8, help='object confidence threshold')\n parser.add_argument('--depth_weight', type=float, default=1, help='weight of depth feature while concatenating')\n parser.add_argument('--nms_thresh', type=float, default=0.56, help='iou thresshold for non-maximum suppression')\n parser.add_argument('--n_cpu', type=int, default=4, help='number of cpu threads to use during batch generation')\n parser.add_argument('--use_cuda', type=bool, default=True, help='whether to use cuda if available')\n parser.add_argument('-p', '--point_cloud', action='store_false', help='Use to disable pointcloud')\n parser.add_argument('-o', '--optical_flow_initiation', action='store_false', help='Use to enable optical flow based velocity initiation')\n parser.add_argument('-q', '--perfect', action='store_true', help='whether to use perfect assignments')\n parser.add_argument('-g', '--ground_truth', action='store_true', help='whether to use ground truth detections')\n parser.add_argument('-r', '--reference', action='store_false', help='whether to use reference detections')\n parser.add_argument('-t', '--track_3d', action='store_true', help='whether to do 3d tracking')\n parser.add_argument('--ref_det', type = str, default = 'new_rrc_subcnn_car', help='lsvm, subcnn, regionlets, maskrcnn')\n parser.add_argument(\"--nn_budget\", type=int, default=100, help=\"Maximum size of the appearance descriptors gallery. If None, no budget is enforced.\")\n parser.add_argument(\"--dummy_node_cost_app\", type=float, default=0.99, help=\"Dummy node appearance cost for JPDA (or maximum distnce when using deepsort)\")\n parser.add_argument(\"--dummy_node_cost_iou\", type=float, default=0.97, help=\"Dummy node iou cost for JPDA (or maximum distnce when using deepsort)\")\n parser.add_argument(\"-c\", \"--combine_features\", action = 'store_false', help=\"Whether to use trained MLP to combine features\")\n parser.add_argument(\"-f\", \"--fpointnet\", action = 'store_false', help=\"Whether to use F-PointNet for 3d detection\")\n parser.add_argument(\"--combo_model\", default = 'weights/resnet_reid_fpointnet_combo_car/mlp__1570759353.0113978/best_checkpoint.tar\"', help=\"Trained MLP checkpoint to combine features\")\n parser.add_argument(\"-j\", \"--JPDA\", action = 'store_false', help=\"Whether to use JPDA for soft assignments\")\n parser.add_argument(\"-l\", \"--LSTM\", action = 'store_true', help=\"Whether to use LSTM for feature combination and update\")\n parser.add_argument(\"--lstm_model\", default = 'weights/aligned_reid_fpointnet_combo/lstm/best_checkpoint.tar', help=\"Trained LSTM checkpoint to combine features\")\n parser.add_argument(\"-m\",\"--m_best_sol\", type=int, default=10, help=\"Number of solutions for JPDA\")\n parser.add_argument(\"--log_data\", action='store_true', help=\"Turn on full data logging\")\n parser.add_argument(\"--max_age\", type=int, default=2, help=\"Number of misses before termination\")\n parser.add_argument(\"--n_init\", type=int, default=2, help=\"Consecutive frames for tentative->confirmed\")\n parser.add_argument(\"--assn_thresh\", type=float, default=0.65, help=\"min prob for match\")\n parser.add_argument(\"--matching_strategy\", type=str, default=\"hungarian\", help=\"matching strategy for JPDA (max_and_threshold, strict_max_pair, or hungarian)\")\n parser.add_argument(\"--kf_appearance_feature\", type=bool, default=False, help=\"Whether to use kf state for apperance features\")\n parser.add_argument('-i', \"--use_imm\", action = 'store_true', help='Whether to use IMM')\n parser.add_argument('-v', \"--verbose\", action = 'store_true', help='Verbose')\n parser.add_argument('--kf_process', type=float, default=5.2, help='kf 2d process noise factor')\n parser.add_argument('--kf_2d_meas', type=float, default=3.2, help='kf 2d measurement noise factor')\n parser.add_argument('--kf_3d_meas', type=float, default=0.25, help='kf 3d measurement noise factor')\n parser.add_argument('--pos_weight_3d', type=float, default=1, help='Weight on position covariance process noise in KF')\n parser.add_argument('--pos_weight', type=float, default=0.006, help='Weight on position covariance process noise in KF')\n parser.add_argument('--vel_weight', type=float, default=0.008, help='Weight on velocity covariance process noise in KF')\n parser.add_argument('--theta_weight', type=float, default=0.02, help='Weight on velocity covariance process noise in KF')\n parser.add_argument('--gate_limit', type=float, default=600, help='Maximum covariance value of the gate')\n parser.add_argument('--initial_uncertainty', type=float, default=1, help='Uncertainty scaling for initial covariance of track')\n parser.add_argument('--uncertainty_limit', type=float, default=1.5, help='Uncertainty limit at which to terminate tracks')\n parser.add_argument(\"--gate_full_state\", action='store_true', help=\"Whether to gate on full kalman state, default is only position\")\n parser.add_argument(\"--near_online\", action = 'store_true', help=\"Whether to do near online tracking\")\n parser.add_argument(\"--omni\", action = 'store_true', help=\"Omni directional camera (JRDB)\")\n opt = parser.parse_args()\n opt.sequence_folder = opt.sequence_folder.rstrip(os.sep)\n opt.using_cuda = torch.cuda.is_available() and opt.use_cuda\n if not opt.point_cloud and opt.track_3d:\n raise(\"Must provide point cloud if doing 3D tracking!\")\n if opt.verbose:\n print(opt)\n if not os.path.exists(opt.output_folder):\n os.makedirs(opt.output_folder)\n return opt\n\n# @profile\ndef main(opt):\n\n if opt.verbose:\n print(\"------------------------\")\n print(\"RUNNING SET UP\")\n print(\"------------------------\")\n tf.logging.set_verbosity(40)\n random.seed(0)\n Tensor = torch.cuda.FloatTensor if opt.using_cuda else torch.FloatTensor\n os.makedirs(opt.output_folder, exist_ok=True)\n if opt.LSTM:\n opt.max_cosine_distance = 1\n lstm = CombiLSTM()\n checkpoint = torch.load(opt.lstm_model)\n lstm.load_state_dict(checkpoint['state_dict'])\n if opt.using_cuda:\n lstm.cuda()\n lstm.eval()\n else:\n lstm = None\n if opt.combine_features:\n combination_model = CombiNet()\n checkpoint = torch.load(opt.combo_model)\n combination_model.load_state_dict(checkpoint['state_dict'])\n if opt.using_cuda:\n combination_model.cuda()\n combination_model.eval()\n else:\n combination_model = None\n \n dataset = SequenceDataset(opt.sequence_folder, point_cloud=opt.point_cloud, omni=opt.omni)\n dataloader = DataLoader(dataset, batch_size=1, shuffle=False, num_workers=opt.n_cpu, collate_fn = collate_fn)\n appearance_model = create_appearance_model(opt.appearance_model, opt.aligned_reid_ckpt, opt.resnet_reid_ckpt, opt.using_cuda)\n if opt.point_cloud:\n depth_model = create_depth_model(opt.depth_model, opt.depth_config_path)\n if opt.track_3d:\n tracker = Tracker_3d(appearance_model=appearance_model, cuda=opt.using_cuda, JPDA = opt.JPDA, m_best_sol=opt.m_best_sol,\n max_age = opt.max_age, n_init=opt.n_init, assn_thresh=opt.assn_thresh,\n matching_strategy=opt.matching_strategy,\n gate_full_state=opt.gate_full_state,\n kf_vel_params=(opt.pos_weight_3d, opt.pos_weight, opt.vel_weight, opt.theta_weight,\n opt.kf_process, opt.kf_2d_meas, opt.kf_3d_meas, opt.initial_uncertainty),\n calib=dataset.calib,\n dummy_node_cost_iou=opt.dummy_node_cost_iou,\n dummy_node_cost_app=opt.dummy_node_cost_app,\n nn_budget=opt.nn_budget,\n use_imm=opt.use_imm,\n uncertainty_limit=opt.uncertainty_limit,\n gate_limit=opt.gate_limit,\n omni=opt.omni)\n else:\n tracker = Tracker(appearance_model=appearance_model, cuda=opt.using_cuda, JPDA = opt.JPDA, m_best_sol=opt.m_best_sol,\n max_age = opt.max_age, n_init=opt.n_init, assn_thresh=opt.assn_thresh,\n matching_strategy=opt.matching_strategy,\n kf_appearance_feature=opt.kf_appearance_feature,\n gate_full_state=opt.gate_full_state,\n kf_vel_params=(opt.pos_weight, opt.vel_weight, opt.kf_process, opt.kf_2d_meas, opt.initial_uncertainty),\n kf_walk_params=(opt.pos_weight, opt.vel_weight, opt.kf_process, opt.kf_2d_meas, opt.initial_uncertainty),\n calib=dataset.calib,\n dummy_node_cost_iou=opt.dummy_node_cost_iou,\n dummy_node_cost_app=opt.dummy_node_cost_app,\n nn_budget=opt.nn_budget,\n use_imm=opt.use_imm,\n uncertainty_limit=opt.uncertainty_limit,\n optical_flow=opt.optical_flow_initiation,\n gate_limit=opt.gate_limit)\n\n results = []\n results_3d = []\n n_frames = len(dataloader)\n if opt.log_data:\n full_log = [{'tracks':[], 'detections':[], 'detections_3d':[]} for _ in range(n_frames)]\n det_matrix = None\n seq_name = os.path.split(opt.sequence_folder)[-1]\n\n frame_times = []\n if opt.verbose:\n print(\"------------------------\")\n print(\"BEGINNING TRACKING OF SEQUENCE %s\"%seq_name)\n print(\"------------------------\")\n for frame_idx, img_path, input_img, point_cloud in tqdm(dataloader, ncols = 100, disable=not opt.verbose):\n # if frame_idx > 120:\n # break\n # elif frame_idx < 98:\n # continue\n\n if opt.log_data:\n full_log[frame_idx]['img_path'] = copy.copy(img_path)\n input_img = input_img.type(Tensor)\n if opt.reference:\n detections, object_ids, det_matrix = read_ground_truth_2d_detections(os.path.join(opt.sequence_folder,'det',opt.ref_det+'.txt'), frame_idx, det_matrix, threshold = 0, nms_threshold = opt.nms_thresh)\n elif opt.ground_truth:\n detections, object_ids, det_matrix = read_ground_truth_2d_detections(os.path.join(opt.sequence_folder,'gt','gt.txt'), frame_idx, det_matrix, nms_threshold = opt.nms_thresh)\n else:\n raise(\"Must specify ground truth or detections\")\n\n # --- START OF TRACKING ---\n # start_time = time.time()\n if detections is None or len(detections)==0:\n tracker.predict()\n if opt.log_data:\n full_log[frame_idx]['predicted_tracks'] = copy.deepcopy(tracker.tracks)\n start_time = time.time()\n tracker.update(input_img, [])\n else:\n total_dets = len(detections)\n patches = get_image_patches(input_img, detections)\n appearance_features = generate_features_batched(appearance_model, patches, opt, object_ids)\n if opt.point_cloud:\n if not opt.omni:\n point_cloud = point_cloud[point_cloud[:,2]>=0]\n if opt.fpointnet:\n boxes_3d, valid_3d, _, scores_3d, depth_features = generate_detections_3d(depth_model, \n detections, np.asarray(point_cloud), \n dataset.calib, input_img.shape,\n peds='ped' in opt.ref_det or opt.omni)\n depth_features = convert_depth_features(depth_features, valid_3d)\n else:\n boxes_3d, valid_3d = read_ground_truth_3d_detections(os.path.join(opt.sequence_folder,'gt','3d_detections.txt'), frame_idx) \n features, appearance_features = combine_features(appearance_features, depth_features, valid_3d, combination_model, depth_weight = opt.depth_weight)\n # boxes_3d = boxes_3d[valid_3d != -1] # Old and buggy way of handling missing box\n # detections = detections[valid_3d != -1]\n if np.any(valid_3d == -1):\n compare_2d = True\n else:\n compare_2d = False\n if len(boxes_3d) > 0:\n detections_3d = []\n for idx, box in enumerate(boxes_3d):\n if valid_3d[idx] == -1:\n detections_3d.append(None)\n else:\n detections_3d.append(np.array(box).astype(np.float32))\n else:\n detections_3d = None\n else:\n appearance_features = [appearance_features[i] for i in range(total_dets)]\n features = [None]*len(appearance_features)\n compare_2d = True\n detections_3d = None\n detections = convert_detections(detections, features, appearance_features, detections_3d)\n tracker.predict()\n if opt.log_data:\n full_log[frame_idx]['predicted_tracks'] = copy.deepcopy(tracker.tracks)\n start_time = time.time()\n tracker.update(input_img, detections, compare_2d)\n\n # --- END OF TRACKING ---\n end_time = time.time()\n frame_times.append(end_time - start_time)\n\n\n if opt.log_data:\n full_tracks = copy.deepcopy(tracker.tracks)\n temp_tracks = []\n for track in full_tracks:\n bbox = track.to_tlwh(None)\n if not (bbox[0] < 0-10 or bbox[1] < 0-10 or bbox[0] + bbox[2] > input_img.shape[2]+10 or bbox[1] + bbox[3] > input_img.shape[1]+10):\n temp_tracks.append(track)\n full_log[frame_idx]['tracks'] = temp_tracks\n full_log[frame_idx]['detections'] = copy.deepcopy(detections)\n\n for track in tracker.tracks:\n if opt.track_3d:\n bbox_3d = track.to_tlwh3d()\n else:\n bbox = track.to_tlwh(None)\n if bbox[0] < 0-10 or bbox[1] < 0-10 or bbox[0] + bbox[2] > input_img.shape[2]+10 or bbox[1] + bbox[3] > input_img.shape[1]+10:\n continue\n bbox[0] = max(0,bbox[0]) # Frame adjustments\n bbox[1] = max(0,bbox[1])\n bbox[2] = min(bbox[0]+bbox[2], input_img.shape[2])-bbox[0]\n bbox[3] = min(bbox[1]+bbox[3], input_img.shape[1])-bbox[1]\n\n track_status = 1\n if not track.is_confirmed(): # or track.time_since_update > 0:\n if opt.near_online:\n if not track.is_confirmed():\n track_status = 0\n else:\n track_status = 2\n continue\n else:\n continue\n if opt.near_online:\n if opt.track_3d:\n results_3d.append([frame_idx, track.track_id, bbox_3d[0], bbox_3d[1], bbox_3d[2], bbox_3d[3], bbox_3d[4], bbox_3d[5], bbox_3d[6], track_status])\n else:\n results.append([frame_idx, track.track_id, bbox[0], bbox[1], bbox[2], bbox[3], track_status])\n\n if track_status == 1: #updates 0s\n for row_i in range(len(results)):\n if results[row_i][1] == track.track_id:\n results[row_i][6] = 1\n if opt.point_cloud:\n if results_3d[row_i][1] == track.track_id:\n results_3d[row_i][7] = 1\n else:\n if opt.track_3d:\n results_3d.append([frame_idx, track.track_id, bbox_3d[0], bbox_3d[1], bbox_3d[2], bbox_3d[3], bbox_3d[4], bbox_3d[5], bbox_3d[6]])\n else:\n results.append([frame_idx, track.track_id, bbox[0], bbox[1], bbox[2], bbox[3]])\n # if opt.point_cloud:\n\n frame_times = np.asarray(frame_times)\n if opt.verbose:\n print(\"------------------------\")\n print(\"COMPLETED TRACKING, SAVING RESULTS\")\n print(\"------------------------\")\n print('\\n\\n','Total Tracking Time:',np.sum(frame_times),'Average Time Per Frame:',np.mean(frame_times))\n\n if opt.track_3d:\n output_file_3d = os.path.join(opt.output_folder, seq_name+\"_3d.txt\")\n if len(results_3d) > 0:\n with open(output_file_3d, 'w+') as f:\n for row in results_3d:\n if opt.near_online and row[9] != 1:\n continue\n print('%d,%d,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.4f,1,1,1,-1' % (\n row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8]), file=f)\n else:\n output_file = os.path.join(opt.output_folder, seq_name+\".txt\")\n if len(results) > 0:\n with open(output_file, 'w+') as f:\n for row in results:\n if opt.near_online and row[6] != 1:\n continue\n print('%d,%d,%.2f,%.2f,%.2f,%.2f,1,1,1,-1' % (\n row[0], row[1], row[2], row[3], row[4], row[5]), file=f)\n\n if opt.log_data:\n output_file = os.path.join(opt.output_folder, seq_name+\".p\")\n with open(output_file, 'wb') as f:\n pickle.dump(full_log, f)\n\nif __name__=='__main__':\n opt = parse_arguments()\n main(opt)\n" }, { "path": "paper_experiments/utils/EKF.py", "content": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\nimport scipy.linalg\nimport pdb\n\n\"\"\"\nTable for the 0.95 quantile of the chi-square distribution with N degrees of\nfreedom (contains values for N=1, ..., 9). Taken from MATLAB/Octave's chi2inv\nfunction and used as Mahalanobis gating threshold.\n\"\"\"\nchi2inv95 = {\n 1: 3.8415,\n 2: 5.9915,\n 3: 7.8147,\n 4: 9.4877,\n 5: 11.070,\n 6: 12.592,\n 7: 14.067,\n 8: 15.507,\n 9: 16.919}\n\nchi2inv90 = {\n 1: 2.706,\n 2: 4.605,\n 3: 6.251,\n 4: 7.779,\n 5: 9.236,\n 6: 10.645,\n 7: 12.017,\n 8: 13.363,\n 9: 14.684}\n\nchi2inv975 = {\n 1: 5.025,\n 2: 7.378,\n 3: 9.348,\n 4: 11.143,\n 5: 12.833,\n 6: 14.449,\n 7: 16.013,\n 8: 17.535,\n 9: 19.023}\n\nchi2inv10 = {\n 1: .016,\n 2: .221,\n 3: .584,\n 4: 1.064,\n 5: 1.610,\n 6: 2.204,\n 7: 2.833,\n 8: 3.490,\n 9: 4.168}\n\n\nchi2inv995 = {\n 1: 0.0000393,\n 2: 0.0100,\n 3: .0717,\n 4: .207,\n 5: .412,\n 6: .676,\n 7: .989,\n 8: 1.344,\n 9: 1.735}\n\n\nchi2inv75 = {\n 1: 1.323,\n 2: 2.773,\n 3: 4.108,\n 4: 5.385,\n 5: 6.626,\n 6: 7.841,\n 7: 9.037,\n 8: 10.22,\n 9: 11.39}\n\ndef squared_mahalanobis_distance(mean, covariance, measurements):\n # cholesky factorization used to solve for \n # z = d * inv(covariance)\n # so z is also the solution to \n # covariance * z = d \n d = measurements - mean\n\n # Note: The cholesky factorization is giving weird answers. This is marginally slower but correct\n return np.matmul(np.matmul(d, np.linalg.inv(covariance)), d.T).diagonal()\n\n # print(\"Measurements:\", measurements)\n # print(\"Mean:\", mean)\n # print(\"dshape:\", d.shape, \"d:\", d)\n # print(\"d*inv(cov)\", np.matmul(d, np.linalg.inv(covariance)))\n\n cholesky_factor = np.linalg.cholesky(covariance)\n z = scipy.linalg.solve_triangular(\n cholesky_factor, d.T, lower=True, check_finite=False,\n overwrite_b=True)\n\n squared_maha = np.sum(z * (measurements-mean).T, axis=0)\n # print(\"Squared maha dist:\", squared_maha)\n # print(\"cov:\", covariance)\n # print(\"z\", z, '\\n')\n return squared_maha\n\n\nclass EKF(object):\n \"\"\"\n Generic extended kalman filter class\n\n \"\"\"\n\n def __init__(self):\n pass\n\n def initiate(self, measurement):\n \"\"\"Create track from unassociated measurement.\n\n Parameters\n ----------\n measurement : ndarray\n\n Returns\n -------\n (ndarray, ndarray)\n Returns the mean vector and covariance matrix of the new track. \n Unobserved velocities are initialized to 0 mean.\n\n \"\"\"\n pass\n\n\n def predict_mean(self, mean):\n # Updates predicted state from previous state (function g)\n # Calculates motion update Jacobian (Gt)\n # Returns (g(mean), Gt)\n pass\n\n def get_process_noise(self, mean, covariance):\n # Returns Rt the motion noise covariance\n pass\n def predict_covariance(self, mean, covariance):\n pass\n\n def project_mean(self, mean):\n # Measurement prediction from state (function h)\n # Calculations sensor update Jacobian (Ht)\n # Returns (h(mean), Ht)\n pass\n def project_cov(self, mean, covariance):\n pass\n\n def predict(self, mean, covariance, last_detection, next_to_last_detection):\n \"\"\"Run Kalman filter prediction step.\n\n Parameters\n ----------\n mean : ndarray\n The mean vector of the object state at the previous\n time step.\n covariance : ndarray\n The covariance matrix of the object state at the\n previous time step.\n\n Returns\n -------\n (ndarray, ndarray)\n Returns the mean vector and covariance matrix of the predicted\n state. Unobserved velocities are initialized to 0 mean.\n\n \"\"\"\n # Perform prediction\n covariance = self.predict_covariance(mean, covariance, last_detection, next_to_last_detection)\n mean = self.predict_mean(mean)\n\n return mean, covariance\n def get_innovation_cov(self, covariance):\n pass\n\n def project(self, mean, covariance):\n \"\"\"Project state distribution to measurement space.\n\n Parameters\n ----------\n mean : ndarray\n The state's mean vector \n covariance : ndarray\n The state's covariance matrix\n\n Returns\n -------\n (ndarray, ndarray)\n Returns the projected mean and covariance matrix of the given state\n estimate.\n\n \"\"\"\n\n # Measurement uncertainty scaled by estimated height\n return self.project_mean(mean), self.project_cov(mean, covariance)\n\n def update(self, mean, covariance, measurement_t, marginalization=None, JPDA=False):\n \"\"\"Run Kalman filter correction step.\n\n Parameters\n ----------\n mean : ndarray\n The predicted state's mean vector (8 dimensional).\n covariance : ndarray\n The state's covariance matrix (8x8 dimensional).\n measurement : ndarray\n The 4 dimensional measurement vector (x, y, a, h), where (x, y)\n is the center position, a the aspect ratio, and h the height of the\n bounding box.\n\n Returns\n -------\n (ndarray, ndarray)\n Returns the measurement-corrected state distribution.\n\n \"\"\"\n predicted_measurement, innovation_cov = self.project(mean, covariance)\n # cholesky factorization used to solve for kalman gain since\n # K = covariance * update_mat.T * inv(innovation_cov)\n # so K is also the solution to \n # innovation_cov * K = covariance * update_mat.T\n try:\n chol_factor, lower = scipy.linalg.cho_factor(\n innovation_cov, lower=True, check_finite=False)\n kalman_gain = scipy.linalg.cho_solve(\n (chol_factor, lower), np.dot(covariance, self._observation_mat.T).T,\n check_finite=False).T\n except:\n # in case cholesky factorization fails, revert to standard solver\n kalman_gain = np.linalg.solve(innovation_cov, np.dot(covariance, self._observation_mat.T).T).T\n\n if JPDA:\n # marginalization\n innovation = np.zeros((self.ndim)) \n cov_soft = np.zeros((self.ndim, self.ndim))\n\n for measurement_idx, measurement in enumerate(measurement_t):\n\n p_ij = marginalization[measurement_idx + 1] # + 1 for dummy\n y_ij = measurement - predicted_measurement\n innovation += y_ij * p_ij\n cov_soft += p_ij * np.outer(y_ij, y_ij)\n\n cov_soft = cov_soft - np.outer(innovation, innovation)\n\n P_star = covariance - np.linalg.multi_dot((\n kalman_gain, innovation_cov, kalman_gain.T))\n\n p_0 = marginalization[0]\n P_0 = p_0 * covariance + (1 - p_0) * P_star\n\n new_covariance = P_0 + np.linalg.multi_dot((kalman_gain, cov_soft, kalman_gain.T))\n \n else:\n innovation = measurement_t - predicted_measurement\n\n new_covariance = covariance - np.linalg.multi_dot((\n kalman_gain, innovation_cov, kalman_gain.T))\n\n new_mean = mean + np.dot(innovation, kalman_gain.T)\n return new_mean, new_covariance\n" }, { "path": "paper_experiments/utils/JPDA_matching.py", "content": "# vim: expandtab:ts=4:sw=4\nfrom __future__ import absolute_import\nimport numpy as np\nfrom linear_assignment import min_marg_matching\nimport pdb\n\n\ndef get_unmatched(all_idx, matches, i, marginalization=None):\n assigned = [match[i] for match in matches]\n unmatched = set(all_idx) - set(assigned)\n if marginalization is not None:\n # from 1 for dummy node\n in_gate_dets = np.nonzero(np.sum(\n marginalization[:, 1:], axis=0))[0].tolist()\n # unmatched = [d for d in unmatched if d not in in_gate_dets] # TODO: Filter by gate?\n return list(unmatched)\n\n\nclass Matcher:\n\n def __init__(self, detections, marginalizations, confirmed_tracks,\n matching_strategy,\n assignment_threshold=None):\n self.detections = detections\n self.marginalizations = marginalizations\n self.confirmed_tracks = confirmed_tracks\n self.assignment_threshold = assignment_threshold\n self.detection_indices = np.arange(len(detections))\n self.matching_strategy = matching_strategy\n\n def match(self):\n self.get_matches()\n self.get_unmatched_tracks()\n self.get_unmatched_detections()\n return self.matches, self.unmatched_tracks, self.unmatched_detections\n\n def get_matches(self):\n\n if self.matching_strategy == \"max_and_threshold\":\n self.max_and_threshold_matching()\n elif self.matching_strategy == \"hungarian\":\n self.hungarian()\n elif self.matching_strategy == \"max_match\":\n self.max_match()\n elif self.matching_strategy == \"none\":\n self.matches = []\n else: \n raise Exception('Unrecognized matching strategy: {}'.\n format(self.matching_strategy))\n\n def get_unmatched_tracks(self):\n self.unmatched_tracks = get_unmatched(self.confirmed_tracks,\n self.matches, 0)\n\n def get_unmatched_detections(self):\n self.unmatched_detections = get_unmatched(self.detection_indices, self.matches, 1, self.marginalizations)\n\n def max_match(self):\n self.matches = []\n if self.marginalizations.shape[0] == 0:\n return\n\n detection_map = {}\n for i, track_idx in enumerate(self.confirmed_tracks):\n marginalization = self.marginalizations[i,:]\n detection_id = np.argmax(marginalization) - 1 # subtract one for dummy\n\n if detection_id < 0:\n continue\n\n if detection_id not in detection_map.keys():\n detection_map[detection_id] = track_idx\n else:\n cur_track = detection_map[detection_id]\n track_update = track_idx if self.marginalizations[track_idx, detection_id] > self.marginalizations[cur_track, detection_id] else cur_track\n detection_map[detection_id] = track_update\n threshold_p = marginalization[detection_id + 1]\n if threshold_p < self.assignment_threshold:\n continue\n\n for detection in detection_map.keys():\n self.matches.append((detection_map[detection], detection))\n\n def max_and_threshold_matching(self):\n\n self.matches = []\n if self.marginalizations.shape[0] == 0:\n return\n\n for i, track_idx in enumerate(self.confirmed_tracks):\n marginalization = self.marginalizations[i,:]\n detection_id = np.argmax(marginalization) - 1 # subtract one for dummy\n\n if detection_id < 0:\n continue\n\n threshold_p = marginalization[detection_id + 1]\n if threshold_p < self.assignment_threshold:\n continue\n\n self.matches.append((track_idx, detection_id))\n\n def hungarian(self):\n self.matches, _, _ = min_marg_matching(self.marginalizations,\n self.confirmed_tracks,\n self.assignment_threshold)\n \n" }, { "path": "paper_experiments/utils/aligned_reid_utils.py", "content": "from __future__ import print_function\nimport os\nimport os.path as osp\nimport pickle\nfrom scipy import io\nimport datetime\nimport time\nfrom contextlib import contextmanager\nimport numpy as np\nfrom PIL import Image\nimport torch\nfrom torch.autograd import Variable\nfrom models.aligned_reid_model import Model as aligned_reid_model\nfrom models.deep_sort_model import ImageEncoder as deep_sort_model\nfrom utils.resnet_reid_utils import ResNet_Loader\n\ndef time_str(fmt=None):\n if fmt is None:\n fmt = '%Y-%m-%d_%H:%M:%S'\n return datetime.datetime.today().strftime(fmt)\n\n\ndef load_pickle(path):\n \"\"\"Check and load pickle object.\n According to this post: https://stackoverflow.com/a/41733927, cPickle and \n disabling garbage collector helps with loading speed.\"\"\"\n assert osp.exists(path)\n # gc.disable()\n with open(path, 'rb') as f:\n ret = pickle.load(f)\n # gc.enable()\n return ret\n\n\ndef save_pickle(obj, path):\n \"\"\"Create dir and save file.\"\"\"\n may_make_dir(osp.dirname(osp.abspath(path)))\n with open(path, 'wb') as f:\n pickle.dump(obj, f, protocol=2)\n\n\ndef save_mat(ndarray, path):\n \"\"\"Save a numpy ndarray as .mat file.\"\"\"\n io.savemat(path, dict(ndarray=ndarray))\n\n\ndef to_scalar(vt):\n \"\"\"Transform a length-1 pytorch Variable or Tensor to scalar. \n Suppose tx is a torch Tensor with shape tx.size() = torch.Size([1]), \n then npx = tx.cpu().numpy() has shape (1,), not 1.\"\"\"\n if isinstance(vt, Variable):\n return vt.data.cpu().numpy().flatten()[0]\n if torch.is_tensor(vt):\n return vt.cpu().numpy().flatten()[0]\n raise TypeError('Input should be a variable or tensor')\n\n\ndef transfer_optim_state(state, device_id=-1):\n \"\"\"Transfer an optimizer.state to cpu or specified gpu, which means \n transferring tensors of the optimizer.state to specified device. \n The modification is in place for the state.\n Args:\n state: An torch.optim.Optimizer.state\n device_id: gpu id, or -1 which means transferring to cpu\n \"\"\"\n for key, val in state.items():\n if isinstance(val, dict):\n transfer_optim_state(val, device_id=device_id)\n elif isinstance(val, Variable):\n raise RuntimeError(\"Oops, state[{}] is a Variable!\".format(key))\n elif isinstance(val, torch.nn.Parameter):\n raise RuntimeError(\"Oops, state[{}] is a Parameter!\".format(key))\n else:\n try:\n if device_id == -1:\n state[key] = val.cpu()\n else:\n state[key] = val.cuda(device=device_id)\n except:\n pass\n\n\ndef may_transfer_optims(optims, device_id=-1):\n \"\"\"Transfer optimizers to cpu or specified gpu, which means transferring \n tensors of the optimizer to specified device. The modification is in place \n for the optimizers.\n Args:\n optims: A list, which members are either torch.nn.optimizer or None.\n device_id: gpu id, or -1 which means transferring to cpu\n \"\"\"\n for optim in optims:\n if isinstance(optim, torch.optim.Optimizer):\n transfer_optim_state(optim.state, device_id=device_id)\n\n\ndef may_transfer_modules_optims(modules_and_or_optims, device_id=-1):\n \"\"\"Transfer optimizers/modules to cpu or specified gpu.\n Args:\n modules_and_or_optims: A list, which members are either torch.nn.optimizer \n or torch.nn.Module or None.\n device_id: gpu id, or -1 which means transferring to cpu\n \"\"\"\n for item in modules_and_or_optims:\n if isinstance(item, torch.optim.Optimizer):\n transfer_optim_state(item.state, device_id=device_id)\n elif isinstance(item, torch.nn.Module):\n if device_id == -1:\n item.cpu()\n else:\n item.cuda(device=device_id)\n elif item is not None:\n print('[Warning] Invalid type {}'.format(item.__class__.__name__))\n\n\nclass TransferVarTensor(object):\n \"\"\"Return a copy of the input Variable or Tensor on specified device.\"\"\"\n\n def __init__(self, device_id=-1):\n self.device_id = device_id\n\n def __call__(self, var_or_tensor):\n return var_or_tensor.cpu() if self.device_id == -1 \\\n else var_or_tensor.cuda(self.device_id)\n\n\nclass TransferModulesOptims(object):\n \"\"\"Transfer optimizers/modules to cpu or specified gpu.\"\"\"\n\n def __init__(self, device_id=-1):\n self.device_id = device_id\n\n def __call__(self, modules_and_or_optims):\n may_transfer_modules_optims(modules_and_or_optims, self.device_id)\n\n\ndef set_devices(sys_device_ids):\n \"\"\"\n It sets some GPUs to be visible and returns some wrappers to transferring \n Variables/Tensors and Modules/Optimizers.\n Args:\n sys_device_ids: a tuple; which GPUs to use\n e.g. sys_device_ids = (), only use cpu\n sys_device_ids = (3,), use the 4th gpu\n sys_device_ids = (0, 1, 2, 3,), use first 4 gpus\n sys_device_ids = (0, 2, 4,), use the 1st, 3rd and 5th gpus\n Returns:\n TVT: a `TransferVarTensor` callable\n TMO: a `TransferModulesOptims` callable\n \"\"\"\n # Set the CUDA_VISIBLE_DEVICES environment variable\n import os\n visible_devices = ''\n for i in sys_device_ids:\n visible_devices += '{}, '.format(i)\n os.environ['CUDA_VISIBLE_DEVICES'] = visible_devices\n # Return wrappers.\n # Models and user defined Variables/Tensors would be transferred to the\n # first device.\n device_id = 0 if len(sys_device_ids) > 0 else -1\n TVT = TransferVarTensor(device_id)\n TMO = TransferModulesOptims(device_id)\n return TVT, TMO\n\n\ndef set_devices_for_ml(sys_device_ids):\n \"\"\"This version is for mutual learning.\n \n It sets some GPUs to be visible and returns some wrappers to transferring \n Variables/Tensors and Modules/Optimizers.\n \n Args:\n sys_device_ids: a tuple of tuples; which devices to use for each model, \n len(sys_device_ids) should be equal to number of models. Examples:\n \n sys_device_ids = ((-1,), (-1,))\n the two models both on CPU\n sys_device_ids = ((-1,), (2,))\n the 1st model on CPU, the 2nd model on GPU 2\n sys_device_ids = ((3,),)\n the only one model on the 4th gpu \n sys_device_ids = ((0, 1), (2, 3))\n the 1st model on GPU 0 and 1, the 2nd model on GPU 2 and 3\n sys_device_ids = ((0,), (0,))\n the two models both on GPU 0\n sys_device_ids = ((0,), (0,), (1,), (1,))\n the 1st and 2nd model on GPU 0, the 3rd and 4th model on GPU 1\n \n Returns:\n TVTs: a list of `TransferVarTensor` callables, one for one model.\n TMOs: a list of `TransferModulesOptims` callables, one for one model.\n relative_device_ids: a list of lists; `sys_device_ids` transformed to \n relative ids; to be used in `DataParallel`\n \"\"\"\n import os\n\n all_ids = []\n for ids in sys_device_ids:\n all_ids += ids\n unique_sys_device_ids = list(set(all_ids))\n unique_sys_device_ids.sort()\n if -1 in unique_sys_device_ids:\n unique_sys_device_ids.remove(-1)\n\n # Set the CUDA_VISIBLE_DEVICES environment variable\n\n visible_devices = ''\n for i in unique_sys_device_ids:\n visible_devices += '{}, '.format(i)\n os.environ['CUDA_VISIBLE_DEVICES'] = visible_devices\n\n # Return wrappers\n\n relative_device_ids = []\n TVTs, TMOs = [], []\n for ids in sys_device_ids:\n relative_ids = []\n for id in ids:\n if id != -1:\n id = find_index(unique_sys_device_ids, id)\n relative_ids.append(id)\n relative_device_ids.append(relative_ids)\n\n # Models and user defined Variables/Tensors would be transferred to the\n # first device.\n TVTs.append(TransferVarTensor(relative_ids[0]))\n TMOs.append(TransferModulesOptims(relative_ids[0]))\n return TVTs, TMOs, relative_device_ids\n\n\ndef load_ckpt(modules_optims, ckpt_file, load_to_cpu=True, verbose=True):\n \"\"\"Load state_dict's of modules/optimizers from file.\n Args:\n modules_optims: A list, which members are either torch.nn.optimizer \n or torch.nn.Module.\n ckpt_file: The file path.\n load_to_cpu: Boolean. Whether to transform tensors in modules/optimizers \n to cpu type.\n \"\"\"\n map_location = (lambda storage, loc: storage) if load_to_cpu else None\n ckpt = torch.load(ckpt_file, map_location=map_location)\n for m, sd in zip(modules_optims, ckpt['state_dicts']):\n if 'fc.weight' in sd:\n del sd['fc.weight']\n if 'fc.bias' in sd: \n del sd['fc.bias']\n load_state_dict(m, sd)\n if verbose:\n print('Resume from ckpt {}, \\nepoch {}, \\nscores {}'.format(\n ckpt_file, ckpt['ep'], ckpt['scores']))\n return ckpt['ep'], ckpt['scores']\n\n\ndef save_ckpt(modules_optims, ep, scores, ckpt_file):\n \"\"\"Save state_dict's of modules/optimizers to file. \n Args:\n modules_optims: A list, which members are either torch.nn.optimizer \n or torch.nn.Module.\n ep: the current epoch number\n scores: the performance of current model\n ckpt_file: The file path.\n Note:\n torch.save() reserves device type and id of tensors to save, so when \n loading ckpt, you have to inform torch.load() to load these tensors to \n cpu or your desired gpu, if you change devices.\n \"\"\"\n state_dicts = [m.state_dict() for m in modules_optims]\n ckpt = dict(state_dicts=state_dicts,\n ep=ep,\n scores=scores)\n may_make_dir(osp.dirname(osp.abspath(ckpt_file)))\n torch.save(ckpt, ckpt_file)\n\n\ndef load_state_dict(model, src_state_dict):\n \"\"\"Copy parameters and buffers from `src_state_dict` into `model` and its \n descendants. The `src_state_dict.keys()` NEED NOT exactly match \n `model.state_dict().keys()`. For dict key mismatch, just\n skip it; for copying error, just output warnings and proceed.\n\n Arguments:\n model: A torch.nn.Module object. \n src_state_dict (dict): A dict containing parameters and persistent buffers.\n Note:\n This is modified from torch.nn.modules.module.load_state_dict(), to make\n the warnings and errors more detailed.\n \"\"\"\n from torch.nn import Parameter\n dest_state_dict = model.state_dict()\n for name, param in src_state_dict.items():\n ### CHANGED HERE FOR FINE TUNING\n if name not in dest_state_dict:\n continue\n if isinstance(param, Parameter):\n # backwards compatibility for serialized parameters\n param = param.data\n try:\n dest_state_dict[name].copy_(param)\n except Exception as e:\n print(\"Warning: Error occurs when copying '{}': {}\"\n .format(name, str(e)))\n\n # src_missing = set(dest_state_dict.keys()) - set(src_state_dict.keys())\n # if len(src_missing) > 0:\n # print(\"Keys not found in source state_dict: \")\n # for n in src_missing:\n # print('\\t', n)\n\n # dest_missing = set(src_state_dict.keys()) - set(dest_state_dict.keys())\n # if len(dest_missing) > 0:\n # print(\"Keys not found in destination state_dict: \")\n # for n in dest_missing:\n # print('\\t', n)\n\n\ndef is_iterable(obj):\n return hasattr(obj, '__len__')\n\n\ndef may_set_mode(maybe_modules, mode):\n \"\"\"maybe_modules: an object or a list of objects.\"\"\"\n assert mode in ['train', 'eval']\n if not is_iterable(maybe_modules):\n maybe_modules = [maybe_modules]\n for m in maybe_modules:\n if isinstance(m, torch.nn.Module):\n if mode == 'train':\n m.train()\n else:\n m.eval()\n\n\ndef may_make_dir(path):\n \"\"\"\n Args:\n path: a dir, or result of `osp.dirname(osp.abspath(file_path))`\n Note:\n `osp.exists('')` returns `False`, while `osp.exists('.')` returns `True`!\n \"\"\"\n # This clause has mistakes:\n # if path is None or '':\n\n if path in [None, '']:\n return\n if not osp.exists(path):\n os.makedirs(path)\n\n\nclass AverageMeter(object):\n \"\"\"Modified from Tong Xiao's open-reid. \n Computes and stores the average and current value\"\"\"\n\n def __init__(self):\n self.val = 0\n self.avg = 0\n self.sum = 0\n self.count = 0\n\n def reset(self):\n self.val = 0\n self.avg = 0\n self.sum = 0\n self.count = 0\n\n def update(self, val, n=1):\n self.val = val\n self.sum += val * n\n self.count += n\n self.avg = float(self.sum) / (self.count + 1e-20)\n\n\nclass RunningAverageMeter(object):\n \"\"\"Computes and stores the running average and current value\"\"\"\n\n def __init__(self, hist=0.99):\n self.val = None\n self.avg = None\n self.hist = hist\n\n def reset(self):\n self.val = None\n self.avg = None\n\n def update(self, val):\n if self.avg is None:\n self.avg = val\n else:\n self.avg = self.avg * self.hist + val * (1 - self.hist)\n self.val = val\n\n\nclass RecentAverageMeter(object):\n \"\"\"Stores and computes the average of recent values.\"\"\"\n\n def __init__(self, hist_size=100):\n self.hist_size = hist_size\n self.fifo = []\n self.val = 0\n\n def reset(self):\n self.fifo = []\n self.val = 0\n\n def update(self, val):\n self.val = val\n self.fifo.append(val)\n if len(self.fifo) > self.hist_size:\n del self.fifo[0]\n\n @property\n def avg(self):\n assert len(self.fifo) > 0\n return float(sum(self.fifo)) / len(self.fifo)\n\n\ndef get_model_wrapper(model, multi_gpu):\n from torch.nn.parallel import DataParallel\n if multi_gpu:\n return DataParallel(model)\n else:\n return model\n\n\nclass ReDirectSTD(object):\n \"\"\"Modified from Tong Xiao's `Logger` in open-reid.\n This class overwrites sys.stdout or sys.stderr, so that console logs can\n also be written to file.\n Args:\n fpath: file path\n console: one of ['stdout', 'stderr']\n immediately_visible: If `False`, the file is opened only once and closed\n after exiting. In this case, the message written to file may not be\n immediately visible (Because the file handle is occupied by the\n program?). If `True`, each writing operation of the console will\n open, write to, and close the file. If your program has tons of writing\n operations, the cost of opening and closing file may be obvious. (?)\n Usage example:\n `ReDirectSTD('stdout.txt', 'stdout', False)`\n `ReDirectSTD('stderr.txt', 'stderr', False)`\n NOTE: File will be deleted if already existing. Log dir and file is created\n lazily -- if no message is written, the dir and file will not be created.\n \"\"\"\n\n def __init__(self, fpath=None, console='stdout', immediately_visible=False):\n import sys\n import os\n import os.path as osp\n\n assert console in ['stdout', 'stderr']\n self.console = sys.stdout if console == 'stdout' else sys.stderr\n self.file = fpath\n self.f = None\n self.immediately_visible = immediately_visible\n if fpath is not None:\n # Remove existing log file.\n if osp.exists(fpath):\n os.remove(fpath)\n\n # Overwrite\n if console == 'stdout':\n sys.stdout = self\n else:\n sys.stderr = self\n\n def __del__(self):\n self.close()\n\n def __enter__(self):\n pass\n\n def __exit__(self, *args):\n self.close()\n\n def write(self, msg):\n self.console.write(msg)\n if self.file is not None:\n may_make_dir(os.path.dirname(osp.abspath(self.file)))\n if self.immediately_visible:\n with open(self.file, 'a') as f:\n f.write(msg)\n else:\n if self.f is None:\n self.f = open(self.file, 'w')\n self.f.write(msg)\n\n def flush(self):\n self.console.flush()\n if self.f is not None:\n self.f.flush()\n import os\n os.fsync(self.f.fileno())\n\n def close(self):\n self.console.close()\n if self.f is not None:\n self.f.close()\n\n\ndef set_seed(seed):\n import random\n random.seed(seed)\n print('setting random-seed to {}'.format(seed))\n\n import numpy as np\n np.random.seed(seed)\n print('setting np-random-seed to {}'.format(seed))\n\n import torch\n torch.backends.cudnn.enabled = False\n print('cudnn.enabled set to {}'.format(torch.backends.cudnn.enabled))\n # set seed for CPU\n torch.manual_seed(seed)\n print('setting torch-seed to {}'.format(seed))\n\n\ndef print_array(array, fmt='{:.2f}', end=' '):\n \"\"\"Print a 1-D tuple, list, or numpy array containing digits.\"\"\"\n s = ''\n for x in array:\n s += fmt.format(float(x)) + end\n s += '\\n'\n print(s)\n return s\n\n\n# Great idea from https://github.com/amdegroot/ssd.pytorch\ndef str2bool(v):\n return v.lower() in (\"yes\", \"true\", \"t\", \"1\")\n\n\ndef tight_float_str(x, fmt='{:.4f}'):\n return fmt.format(x).rstrip('0').rstrip('.')\n\n\ndef find_index(seq, item):\n for i, x in enumerate(seq):\n if item == x:\n return i\n return -1\n\n\ndef adjust_lr_exp(optimizer, base_lr, ep, total_ep, start_decay_at_ep):\n \"\"\"Decay exponentially in the later phase of training. All parameters in the \n optimizer share the same learning rate.\n \n Args:\n optimizer: a pytorch `Optimizer` object\n base_lr: starting learning rate\n ep: current epoch, ep >= 1\n total_ep: total number of epochs to train\n start_decay_at_ep: start decaying at the BEGINNING of this epoch\n \n Example:\n base_lr = 2e-4\n total_ep = 300\n start_decay_at_ep = 201\n It means the learning rate starts at 2e-4 and begins decaying after 200 \n epochs. And training stops after 300 epochs.\n \n NOTE: \n It is meant to be called at the BEGINNING of an epoch.\n \"\"\"\n assert ep >= 1, \"Current epoch number should be >= 1\"\n\n if ep < start_decay_at_ep:\n return\n\n for g in optimizer.param_groups:\n g['lr'] = (base_lr * (0.001 ** (float(ep + 1 - start_decay_at_ep)\n / (total_ep + 1 - start_decay_at_ep))))\n print('=====> lr adjusted to {:.10f}'.format(g['lr']).rstrip('0'))\n\n\ndef adjust_lr_staircase(optimizer, base_lr, ep, decay_at_epochs, factor):\n \"\"\"Multiplied by a factor at the BEGINNING of specified epochs. All \n parameters in the optimizer share the same learning rate.\n \n Args:\n optimizer: a pytorch `Optimizer` object\n base_lr: starting learning rate\n ep: current epoch, ep >= 1\n decay_at_epochs: a list or tuple; learning rate is multiplied by a factor \n at the BEGINNING of these epochs\n factor: a number in range (0, 1)\n \n Example:\n base_lr = 1e-3\n decay_at_epochs = [51, 101]\n factor = 0.1\n It means the learning rate starts at 1e-3 and is multiplied by 0.1 at the \n BEGINNING of the 51'st epoch, and then further multiplied by 0.1 at the \n BEGINNING of the 101'st epoch, then stays unchanged till the end of \n training.\n \n NOTE: \n It is meant to be called at the BEGINNING of an epoch.\n \"\"\"\n assert ep >= 1, \"Current epoch number should be >= 1\"\n\n if ep not in decay_at_epochs:\n return\n\n ind = find_index(decay_at_epochs, ep)\n for g in optimizer.param_groups:\n g['lr'] = base_lr * factor ** (ind + 1)\n print('=====> lr adjusted to {:.10f}'.format(g['lr']).rstrip('0'))\n\n\n@contextmanager\ndef measure_time(enter_msg):\n st = time.time()\n print(enter_msg)\n yield\n print('Done, {:.2f}s'.format(time.time() - st))\n\n# @profile\ndef generate_features(appearance_model, patches, opt, object_ids = None):\n Tensor = torch.cuda.FloatTensor if opt.using_cuda else torch.FloatTensor\n features = []\n for i, patch in enumerate(patches):\n if patch is None or patch.nelement()==0:\n features.append(None)\n continue\n patch = patch.unsqueeze(0)\n if opt.perfect:\n feature = torch.zeros(1024)\n feature[object_ids[i]] = 1\n feature = feature.type(Tensor)\n else:\n if opt.appearance_model == 'aligned_reid':\n with torch.no_grad():\n feature ,_ = appearance_model(patch.cuda())\n feature = feature.squeeze(0).type(Tensor)\n elif opt.appearance_model == 'deepsort':\n patch = patch.permute(0,2,3,1).cpu().numpy()\n feature = appearance_model(patch)\n feature = feature[0]\n elif opt.appearance_model == 'resnet_reid':\n patch = patch.permute(0,2,3,1)\n feature = appearance_model.inference([patch.squeeze(0)])\n feature = feature[0][0].type(Tensor)\n features.append(feature)\n return features\n\n# @profile\ndef generate_features_batched(appearance_model, patches, opt, object_ids = None):\n if opt.perfect or opt.appearance_model == 'deepsort': # Do old/slow way if perfect features or deepsort features\n return generate_features(appearance_model, patches, opt, object_ids)\n\n if opt.appearance_model == 'resnet_reid':\n Tensor = torch.cuda.FloatTensor if opt.using_cuda else torch.FloatTensor\n\n features = []\n resnet_patches = []\n for i, patch in enumerate(patches):\n if patch is None or patch.nelement()==0:\n features.append(None)\n else:\n features.append(1)\n resnet_patches.append(patch.permute(1,2,0))\n\n resnet_features = appearance_model.inference(resnet_patches)\n ctr = 0\n for i in range(len(features)):\n if features[i] is not None:\n features[i] = resnet_features[ctr].type(Tensor)\n ctr += 1\n return features\n\n elif opt.appearance_model == 'aligned_reid':\n return generate_features(appearance_model, patches, opt, object_ids) #TODO: Fix batched appearance features. This currently gives bad features\n Tensor = torch.cuda.FloatTensor if opt.using_cuda else torch.FloatTensor\n maxx = -1\n maxy = -1\n idxs = []\n features = []\n for i, patch in enumerate(patches):\n if patch is None or patch.nelement()==0:\n continue\n maxx = max(maxx, patch.size()[1])\n maxy = max(maxy, patch.size()[2])\n idxs.append(i)\n\n if(maxx==-1 and maxy==-1):\n return features\n batch = torch.zeros(len(idxs),3,maxx,maxy).cuda()\n for i, idx in enumerate(idxs):\n patch = patches[idx]\n patchx = patch.size()[1]\n patchy = patch.size()[2]\n batch[i,:,:patchx,:patchy] = patch\n\n with torch.no_grad():\n features_torch, _ = appearance_model(batch)\n features_torch = features_torch.type(Tensor)\n\n i = 0\n ctr = 0\n for idx in idxs:\n while(i < idx):\n features.append(None)\n i+=1\n features.append(features_torch[ctr,:])\n i+=1\n ctr+=1\n while(i conf_threshold]\n\t########\n\n\tdet_frames = detections[:, 0]\n\tdet_confidence = detections[:, 6]\n\tgt_boxes = np.asarray(list(zip(gt[:, 2], gt[:, 3], gt[:, 4], gt[:, 5])))\n\tdet_boxes = np.asarray(list(zip(detections[:, 2], detections[:, 3], detections[:, 4], detections[:, 5])))\n\tout_matrix = []\n\tassigned_ids = []\n\n\tfor frame in np.unique(det_frames):\n\t\tframe_mask_det = det_frames == frame\n\t\tframe_mask_gt = gt_frames == frame\n\t\tgt_ids = gt[frame_mask_gt, 1]\n\t\tframe_gt_boxes = gt_boxes[frame_mask_gt]\n\t\tframe_det_boxes = det_boxes[frame_mask_det]\n\n\t\tfor i, det_box in enumerate(frame_det_boxes):\n\t\t\tiou_list = np.asarray([iou(gt_box, det_box) for gt_box in frame_gt_boxes])\n\t\t\tiou_sorted = np.argsort(iou_list)\n\t\t\tpositive_idx = np.where(iou_list >= iou_threshold)[0]\n\t\t\tif len(positive_idx)==0:\n\t\t\t\tassigned_ids.append(-1)\n\t\t\telse:\n\t\t\t\tassigned_ids.append(gt_ids[iou_sorted[-1]])\n\tassigned_ids = np.expand_dims(np.asarray(assigned_ids), 1)\n\t\n\ttry:\n\t\tout_matrix = np.hstack([np.expand_dims(detections[:,0], 1), assigned_ids, detections[:,2:]])\n\texcept:\n\t\tpdb.set_trace()\n\tnp.savetxt(detection_path, out_matrix, delimiter=',', fmt = '%.2f')\n\n\n\treturn\n\nif __name__=='__main__':\n\tap = []\n\tKITTI_root = 'data/KITTI/sequences'\n\tfor sequence in tqdm(range(21)):\n\t\tassign_detection_id(os.path.join(KITTI_root, '%.4d'%sequence, 'det','rrc_subcnn_car_det.txt'), \n\t\t\t\t\t\t\t\t\t\tos.path.join(KITTI_root, '%.4d'%sequence, 'gt', 'gt_car.txt'))\n" }, { "path": "paper_experiments/utils/calibration.py", "content": "import numpy as np\nimport cv2\nimport os\nimport yaml\nimport torch\nimport pdb\n\nclass Calibration(object):\n ''' Calibration matrices and utils\n 3d XYZ in