Showing preview only (750K chars total). Download the full file or copy to clipboard to get everything.
Repository: StanfordVL/JRMOT_ROS
Branch: master
Commit: ca1e87e51ecf
Files: 100
Total size: 715.3 KB
Directory structure:
gitextract_mbelvxqp/
├── CMakeLists.txt
├── LICENSE
├── README.md
├── calib/
│ ├── cameras.yaml
│ └── defaults.yaml
├── config/
│ └── featurepointnet.cfg
├── launch/
│ └── jpda_tracker.launch
├── msg/
│ ├── __init__.py
│ ├── detection2d_with_feature.msg
│ ├── detection2d_with_feature_array.msg
│ ├── detection3d_with_feature.msg
│ └── detection3d_with_feature_array.msg
├── package.xml
├── paper_experiments/
│ ├── models/
│ │ ├── __init__.py
│ │ ├── aligned_reid_model.py
│ │ ├── combination_model.py
│ │ ├── deep_sort_model.py
│ │ ├── featurepointnet_model.py
│ │ ├── pointnet_model.py
│ │ ├── resnet_reid_models.py
│ │ └── yolo_models.py
│ ├── requirements.txt
│ ├── track.py
│ └── utils/
│ ├── EKF.py
│ ├── JPDA_matching.py
│ ├── aligned_reid_utils.py
│ ├── assign_ids_detections.py
│ ├── calibration.py
│ ├── combine_and_process_detections.py
│ ├── dataset.py
│ ├── deep_sort_utils.py
│ ├── detection.py
│ ├── double_measurement_kf.py
│ ├── evaluate_detections.py
│ ├── featurepointnet_model_util.py
│ ├── featurepointnet_tf_util.py
│ ├── imm.py
│ ├── iou_matching.py
│ ├── kf_2d.py
│ ├── kf_3d.py
│ ├── linear_assignment.py
│ ├── logger.py
│ ├── mbest_ilp.py
│ ├── nn_matching.py
│ ├── pointnet_tf_util.py
│ ├── pointnet_transform_nets.py
│ ├── read_detections.py
│ ├── resnet_reid_utils.py
│ ├── test_jpda.py
│ ├── test_kf/
│ │ ├── .gitignore
│ │ ├── run_kf_test.py
│ │ ├── single_track_4state_test.p.val
│ │ ├── single_track_6state_test.p.val
│ │ ├── two_track_4state_test.p.val
│ │ └── write_kf_test.py
│ ├── track.py
│ ├── track_3d.py
│ ├── tracker.py
│ ├── tracker_3d.py
│ ├── tracking_utils.py
│ ├── visualise.py
│ └── yolo_utils/
│ ├── __init__.py
│ ├── datasets.py
│ ├── parse_config.py
│ └── utils.py
├── requirements.txt
└── src/
├── 3d_detector.py
├── EKF.py
├── JPDA_matching.py
├── __init__.py
├── aligned_reid_model.py
├── aligned_reid_utils.py
├── calibration.py
├── combination_model.py
├── deep_sort_utils.py
├── detection.py
├── distances.py
├── double_measurement_kf.py
├── evaluation/
│ ├── __init__.py
│ ├── distances 2.py
│ └── distances.py
├── featurepointnet_model.py
├── featurepointnet_model_util.py
├── featurepointnet_tf_util.py
├── iou_matching.py
├── kf_2d.py
├── linear_assignment.py
├── mbest_ilp.py
├── nn_matching.py
├── pointnet_model.py
├── template 2.py
├── template.py
├── track_3d 2.py
├── track_3d.py
├── tracker_3d 2.py
├── tracker_3d.py
├── tracker_3d_node 2.py
├── tracker_3d_node.py
├── tracking_utils 2.py
└── tracking_utils.py
================================================
FILE CONTENTS
================================================
================================================
FILE: CMakeLists.txt
================================================
cmake_minimum_required(VERSION 2.8.3)
project(jpda_rospack)
## Compile as C++11, supported in ROS Kinetic and newer
# add_compile_options(-std=c++11)
## Find catkin macros and libraries
## if COMPONENTS list like find_package(catkin REQUIRED COMPONENTS xyz)
## is used, also find other catkin packages
find_package(catkin REQUIRED COMPONENTS
roscpp
rospy
std_msgs
vision_msgs
message_generation
)
## System dependencies are found with CMake's conventions
# find_package(Boost REQUIRED COMPONENTS system)
## Uncomment this if the package has a setup.py. This macro ensures
## modules and global scripts declared therein get installed
## See http://ros.org/doc/api/catkin/html/user_guide/setup_dot_py.html
# catkin_python_setup()
################################################
## Declare ROS messages, services and actions ##
################################################
## To declare and build messages, services or actions from within this
## package, follow these steps:
## * Let MSG_DEP_SET be the set of packages whose message types you use in
## your messages/services/actions (e.g. std_msgs, actionlib_msgs, ...).
## * In the file package.xml:
## * add a build_depend tag for "message_generation"
## * add a build_depend and a exec_depend tag for each package in MSG_DEP_SET
## * If MSG_DEP_SET isn't empty the following dependency has been pulled in
## but can be declared for certainty nonetheless:
## * add a exec_depend tag for "message_runtime"
## * In this file (CMakeLists.txt):
## * add "message_generation" and every package in MSG_DEP_SET to
## find_package(catkin REQUIRED COMPONENTS ...)
## * add "message_runtime" and every package in MSG_DEP_SET to
## catkin_package(CATKIN_DEPENDS ...)
## * uncomment the add_*_files sections below as needed
## and list every .msg/.srv/.action file to be processed
## * uncomment the generate_messages entry below
## * add every package in MSG_DEP_SET to generate_messages(DEPENDENCIES ...)
## Generate messages in the 'msg' folder
add_message_files(
FILES
detection2d_with_feature.msg
detection2d_with_feature_array.msg
detection3d_with_feature.msg
detection3d_with_feature_array.msg
)
## Generate services in the 'srv' folder
# add_service_files(
# FILES
# Service1.srv
# Service2.srv
# )
## Generate actions in the 'action' folder
# add_action_files(
# FILES
# Action1.action
# Action2.action
# )
## Generate added messages and services with any dependencies listed here
generate_messages(
DEPENDENCIES
std_msgs
vision_msgs
jpda_rospack
)
################################################
## Declare ROS dynamic reconfigure parameters ##
################################################
## To declare and build dynamic reconfigure parameters within this
## package, follow these steps:
## * In the file package.xml:
## * add a build_depend and a exec_depend tag for "dynamic_reconfigure"
## * In this file (CMakeLists.txt):
## * add "dynamic_reconfigure" to
## find_package(catkin REQUIRED COMPONENTS ...)
## * uncomment the "generate_dynamic_reconfigure_options" section below
## and list every .cfg file to be processed
## Generate dynamic reconfigure parameters in the 'cfg' folder
# generate_dynamic_reconfigure_options(
# cfg/DynReconf1.cfg
# cfg/DynReconf2.cfg
# )
###################################
## catkin specific configuration ##
###################################
## The catkin_package macro generates cmake config files for your package
## Declare things to be passed to dependent projects
## INCLUDE_DIRS: uncomment this if your package contains header files
## LIBRARIES: libraries you create in this project that dependent projects also need
## CATKIN_DEPENDS: catkin_packages dependent projects also need
## DEPENDS: system dependencies of this project that dependent projects also need
catkin_package(
# INCLUDE_DIRS include
# LIBRARIES jpda_rospack
CATKIN_DEPENDS roscpp rospy std_msgs vision_msgs message_runtime
# DEPENDS system_lib
)
###########
## Build ##
###########
## Specify additional locations of header files
## Your package locations should be listed before other locations
include_directories(
# include
${catkin_INCLUDE_DIRS}
)
## Declare a C++ library
# add_library(${PROJECT_NAME}
# src/${PROJECT_NAME}/jpda_rospack.cpp
# )
## Add cmake target dependencies of the library
## as an example, code may need to be generated before libraries
## either from message generation or dynamic reconfigure
# add_dependencies(${PROJECT_NAME} ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS})
## Declare a C++ executable
## With catkin_make all packages are built within a single CMake context
## The recommended prefix ensures that target names across packages don't collide
# add_executable(${PROJECT_NAME}_node src/jpda_rospack_node.cpp)
## Rename C++ executable without prefix
## The above recommended prefix causes long target names, the following renames the
## target back to the shorter version for ease of user use
## e.g. "rosrun someones_pkg node" instead of "rosrun someones_pkg someones_pkg_node"
# set_target_properties(${PROJECT_NAME}_node PROPERTIES OUTPUT_NAME node PREFIX "")
## Add cmake target dependencies of the executable
## same as for the library above
# add_dependencies(${PROJECT_NAME}_node ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS})
## Specify libraries to link a library or executable target against
# target_link_libraries(${PROJECT_NAME}_node
# ${catkin_LIBRARIES}
# )
#############
## Install ##
#############
# all install targets should use catkin DESTINATION variables
# See http://ros.org/doc/api/catkin/html/adv_user_guide/variables.html
## Mark executable scripts (Python etc.) for installation
## in contrast to setup.py, you can choose the destination
# install(PROGRAMS
# scripts/my_python_script
# DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION}
# )
## Mark executables and/or libraries for installation
# install(TARGETS ${PROJECT_NAME} ${PROJECT_NAME}_node
# ARCHIVE DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION}
# LIBRARY DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION}
# RUNTIME DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION}
# )
## Mark cpp header files for installation
# install(DIRECTORY include/${PROJECT_NAME}/
# DESTINATION ${CATKIN_PACKAGE_INCLUDE_DESTINATION}
# FILES_MATCHING PATTERN "*.h"
# PATTERN ".svn" EXCLUDE
# )
## Mark other files for installation (e.g. launch and bag files, etc.)
# install(FILES
# # myfile1
# # myfile2
# DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION}
# )
#############
## Testing ##
#############
## Add gtest based cpp test target and link libraries
# catkin_add_gtest(${PROJECT_NAME}-test test/test_jpda_rospack.cpp)
# if(TARGET ${PROJECT_NAME}-test)
# target_link_libraries(${PROJECT_NAME}-test ${PROJECT_NAME})
# endif()
## Add folders to be run by python nosetests
# catkin_add_nosetests(test)
================================================
FILE: LICENSE
================================================
MIT License
Copyright (c) 2020 Stanford Vision and Learning Group
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: README.md
================================================
# JRMOT ROS package
The repository contains the code for the work "JRMOT: A Real-Time 3D Multi-Object Tracker and a New Large-Scale Dataset".
Note that due to the global pandemic, this repository is still a work in progress. Updates will be made as soon as possible.
## Introduction
JRMOT is a 3D multi object tracking system that:
- Is real-time
- Is online
- Fuses 2D and 3D information
- Achieves State of the Art performance on KITTI
We also release JRDB:
- A dataset with over 2 million annotated boxes and 3500 time consistent trajectories in 2D and 3D
- Captured in social, human-centric settings
- Captured by our social mobile-manipulator JackRabbot
- Contains 360 degree cylindrical images, stereo camera images, 3D pointclouds and more sensing modalties
All information, including download links for JRDB can be found [here](https://jrdb.stanford.edu).
## JRMOT
- Our system is built on top of state of the art 2D and 3D detectors (mask-RCNN and F-PointNet respectively). These detections are associated with predicted track locations at every time step.
- Association is done via a novel feature fusion, as well as a cost selection procedure, followed by Kalman state gating and JPDA.
- Given the JPDA output, we use both 2D and 3D detections in a novel multi-modal Kalman filter to update the track locations.
## Using the code
There are 3 nodes forming parts of the ROS package:
+ 3d_detector.py: Runs F-PointNet, which performs 3D detection and 3D feature extraction
+ template.py: Runs Aligned-Re-ID, which performs 2D feature extraction
+ tracker_3d_node.py: Performs tracking while taking both 2D detections + features and 3D detections + features as input
The launch file in the folder "launch" launches all 3 nodes.
## Dependencies
The following are dependencies of the code:
+ 2D detector: The 2D detector is not included in this package. To interface with your own 2D detector, please modify the file template.py to subscribe to the correct topic, and also to handle the conversion from ROS message to numpy array.
+ Spencer People Tracking messages: The final tracker output is in a Spencer People Tracking message. Please install this package and include these message types.
+ Various python packages: These can be found in [requirements.txt](./requirements.txt).. Please install all dependencies prior to running the code (including CUDA and cuDNN. Additionally, this code requires a solver called Gurobi. Instructions to install gurobipy can be found [here](https://www.gurobi.com/documentation/9.0/quickstart_mac/the_grb_python_interface_f.html).
+ Weight files: The trained weights, (trained on JRDB) for FPointNet and Aligne-ReID can be found [here](https://drive.google.com/open?id=1YQinMPVWEI44KezS9inXe0mvVnm4aL3s).
## Citation
If you find this work useful, please cite:
```
@INPROCEEDINGS{shenoi2020jrmot,
author={A. {Shenoi} and M. {Patel} and J. {Gwak} and P. {Goebel} and A. {Sadeghian} and H. {Rezatofighi} and R. {Mart\'in-Mart\'in} and S. {Savarese}},
booktitle={2020 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},
title={JRMOT: A Real-Time 3D Multi-Object Tracker and a New Large-Scale Dataset},
year={2020},
volume={},
number={},
pages={10335-10342},
doi={10.1109/IROS45743.2020.9341635}}
```
If you utilise our dataset, please also cite:
```
@article{martin2019jrdb,
title={JRDB: A dataset and benchmark of egocentric visual perception for navigation in human environments},
author={Mart{\'i}n-Mart{\'i}n, Roberto and Patel, Mihir and Rezatofighi, Hamid and Shenoi, Abhijeet and Gwak, JunYoung and Frankel, Eric and Sadeghian, Amir and Savarese, Silvio},
journal={arXiv preprint arXiv:1910.11792},
year={2019}
}
```
##
================================================
FILE: calib/cameras.yaml
================================================
stitching:
radius: 3360000
rotation: 0
scalewidth: 1831
crop: 1
cameras:
# camera order matters!
sensor_0:
width: 752
height: 480
D: -0.336591 0.159742 0.00012697 -7.22557e-05 -0.0461953
# K = fx 0 cx
# 0 fy cy
# 0 0 1
K: >
476.71 0 350.738
0 479.505 209.532
0 0 1
R: >
0.999994 0.000654539 0.00340293
-0.000654519 1 -6.81963e-06
-0.00340293 4.59231e-06 0.999994
T: -0.0104242 -3.70974 -56.9177
sensor_1:
width: 752
height: 480
D: -0.335073 0.151959 -0.000232061 0.00032014 -0.0396825
K: >
483.254 0 365.33
0 485.78 210.953
0 0 1
R: >
0.305706 -0.00895443 -0.952084
0.0110396 0.999922 -0.00585963
0.952062 -0.0087193 0.305781
T: 0.93957 -4.05131 -52.03
sensor_2:
width: 752
height: 480
D: -0.338469 0.156256 -0.000385467 0.000295485 -0.0401965
K: >
483.911 0 355.144
0 486.466 223.026
0 0 1
R: >
-0.806828 0.0136361 -0.590629
0.00870468 0.999899 0.011194
0.590723 0.00389039 -0.806865
T: -0.25753 -6.54978 -47.7311
sensor_3:
width: 752
height: 480
D: -0.330848 0.14747 8.59247e-05 0.000262599 -0.0385311
K: >
475.807 0 339.53
0 478.371 188.481
0 0 1
R: >
-0.811334 0.0033829 0.584574
0.00046071 0.999987 -0.00514746
-0.584583 -0.00390699 -0.811324
T: 2.72207 -6.82928 -45.9778
sensor_4:
width: 752
height: 480
D: -0.34064 0.168338 0.000147292 0.000229372 -0.0516133
K: >
485.046 0 368.864
0 488.185 208.215
0 0 1
R: >
0.310275 0.00160497 0.950645
-0.00648686 0.999979 0.000428942
-0.950625 -0.00629979 0.310279
T: -0.333857 -5.12974 -56.0573
sensor_5:
width: 752
height: 480
D: -0.338422 0.163703 -0.000376267 7.73351e-06 -0.0479871
K: >
478.406 0 353.499
0 481.322 190.225
0 0 1
R: >
0.999995 0.00282205 0.00163291
-0.00282345 0.999996 0.000852931
-0.00163049 -0.000857537 0.999998
T: -0.903588 -126.851 -56.6256
sensor_6:
width: 752
height: 480
D: -0.340676 0.165511 -0.00035978 0.000181532 -0.0493721
K: >
480.459 0 362.503
0 482.924 197.949
0 0 1
R: >
0.308288 -0.0110391 -0.951229
-0.000933102 0.999929 -0.0119067
0.951293 0.00455829 0.308256
T: 1.74525 -127.214 -51.7722
sensor_7:
width: 752
height: 480
D: -0.344379 0.170343 -0.000137847 0.000141047 -0.0510536
K: >
486.491 0 361.559
0 489.22 210.547
0 0 1
R: >
-0.808201 0.0313998 -0.588068
0.026057 0.999506 0.0175574
0.588329 -0.00113337 -0.808621
T: -2.56535 -129.191 -47.5803
sensor_8:
width: 752
height: 480
D: -0.331228 0.144696 0.000117553 0.000566449 -0.0343506
K: >
476.708 0 354.16
0 479.424 209.383
0 0 1
R: >
-0.807384 -0.00296577 0.590019
-0.0122001 0.999857 -0.0116688
-0.589901 -0.0166195 -0.807305
T: 3.39727 -129.381 -45.2409
sensor_9:
width: 752
height: 480
D: -0.345189 0.180808 0.000276465 0.000131868 -0.062103
K: >
484.219 0 345.303
0 487.312 192.371
0 0 1
R: >
0.308505 0.00370159 0.951215
-0.00403535 0.999988 -0.00258261
-0.951214 -0.00304174 0.308517
T: 0.354966 -128.218 -54.0617
================================================
FILE: calib/defaults.yaml
================================================
calibrated:
# the lidar_to_rgb parameters allow tweaking of the transformation between lidar and rgb frames
# the default transformation is taken from the TF Tree
# NOTE: applied to the original (sensor/velodyne) frame [x forward, y left, z up]:
lidar_upper_to_rgb:
# in meters: [x,y,z]
translation: [0, 0, -0.33529]
# in radians: [x,y,z]
rotation: [0, 0, 0.085]
lidar_lower_to_rgb:
translation: [0, 0, 0.13511]
rotation: [0, 0, 0]
image:
# all in pixels
width: 3760
height: 480
# y-axis forward pixel offset (e.g. 3760/2 => 1880, b/c center of the cylindrical image is forward)
# TODO: move into calibrated params, when auto-calibration is possible
stitched_image_offset: 1880
frames:
# lookup for people transforms
global: base_link
# name of the rgb360 camera frame to which we wish to transform
rgb360: occam
================================================
FILE: config/featurepointnet.cfg
================================================
[general]
num_point = 1024
model_path = /home/sibot/jr2_catkin_ws/src/jpda_rospack/src/fpointnet_jrdb/model.ckpt
================================================
FILE: launch/jpda_tracker.launch
================================================
<?xml version="1.0" encoding="utf-8"?>
<launch>
<!-- Console launch prefix -->
<arg name="output" default="screen"/>
<!-- Config and weights folder. -->
<arg name="aligned_reid_model" default="$(find jpda_rospack)/src/aligned_reid_JRDB_weights.pth"/>
<arg name="fpointnet_config" default="$(find jpda_rospack)/config/featurepointnet.cfg"/>
<arg name="calib_3d" default="$(find jpda_rospack)/calib"/>
<arg name="combination_depth_weight" default="1"/>
<arg name="combination_model_path" default="0"/>
<node pkg="jpda_rospack" type="template.py" name="jpda_aligned_reid" output="$(arg output)" respawn="false">
<param name="aligned_reid_model" value="$(arg aligned_reid_model)" />
</node>
<node pkg="jpda_rospack" type="3d_detector.py" name="jpda_3d_detector" output="$(arg output)" respawn="false">
<param name="fpointnet_config" value="$(arg fpointnet_config)" />
<param name="calib_3d" value="$(arg calib_3d)" />
</node>
<node pkg="jpda_rospack" type="tracker_3d_node.py" name="jpda_tracker_3d" output="$(arg output)" respawn="false">
<param name="combination_depth_weight" value="$(arg combination_depth_weight)" />
<param name="calib_3d" value="$(arg calib_3d)" />
<param name="combination_model_path" value="$(arg combination_model_path)" />
</node>
</launch>
================================================
FILE: msg/__init__.py
================================================
================================================
FILE: msg/detection2d_with_feature.msg
================================================
# This message contains a 2D bounding box corresponding to the detection of a person
# Also contains the feature of this person used for re-ID
Header header #header timestamp is time of frame acquisition
uint64 x1 # x coordinate of the top left of the bounding box
uint64 y1 # y coordinate of the top left of the bounding box
uint64 x2 # x coordinate of the bottom right of the bounding box
uint64 y2 # y coordinate of the bottom right of the bounding box
float64[] feature # re-ID feature
uint8 frame_det_id #unique id of this detection within this frame (used for associating 2D and 3D detections)
bool valid # whether detection is valid (within the boundaries of the image and has minimum required size)
================================================
FILE: msg/detection2d_with_feature_array.msg
================================================
Header header
detection2d_with_feature[] detection2d_with_features
================================================
FILE: msg/detection3d_with_feature.msg
================================================
# This message contains a 3D bounding box corresponding to the detection of a person
# Also contains the feature of this person used for re-ID
Header header #header timestamp is time of frame acquisition
float32 x # x coordinate of the center of the bottom face of the bounding box
float32 y # y coordinate of the center of the bottom face of the bounding box
float32 z # x coordinate of the center of the bottom face of the bounding box
float32 l # size of bounding box along x dimension
float32 h # size of bounding box along y dimension
float32 w # size of bounding box along z dimension
float32 theta # rotation of bounding box with respect to the positive x axis
float64[] feature # re-ID feature
uint8 frame_det_id #unique id of this detection within this frame (used for associating 2D and 3D detections)
bool valid # whether detection is valid (enough lidar points)
================================================
FILE: msg/detection3d_with_feature_array.msg
================================================
Header header
detection3d_with_feature[] detection3d_with_features
================================================
FILE: package.xml
================================================
<?xml version="1.0"?>
<package format="2">
<name>jpda_rospack</name>
<version>0.0.1</version>
<description>The jpda_rospack package</description>
<!-- One maintainer tag required, multiple allowed, one person per tag -->
<!-- Example: -->
<!-- <maintainer email="jane.doe@example.com">Jane Doe</maintainer> -->
<maintainer email="ashenoi@cs.stanford.edu">ashenoi</maintainer>
<!-- One license tag required, multiple allowed, one license per tag -->
<!-- Commonly used license strings: -->
<!-- BSD, MIT, Boost Software License, GPLv2, GPLv3, LGPLv2.1, LGPLv3 -->
<license>TODO</license>
<!-- Url tags are optional, but multiple are allowed, one per tag -->
<!-- Optional attribute type can be: website, bugtracker, or repository -->
<!-- Example: -->
<!-- <url type="website">http://wiki.ros.org/jpda_rospack</url> -->
<!-- Author tags are optional, multiple are allowed, one per tag -->
<!-- Authors do not have to be maintainers, but could be -->
<!-- Example: -->
<!-- <author email="jane.doe@example.com">Jane Doe</author> -->
<!-- The *depend tags are used to specify dependencies -->
<!-- Dependencies can be catkin packages or system dependencies -->
<!-- Examples: -->
<!-- Use depend as a shortcut for packages that are both build and exec dependencies -->
<!-- <depend>roscpp</depend> -->
<!-- Note that this is equivalent to the following: -->
<!-- <build_depend>roscpp</build_depend> -->
<!-- <exec_depend>roscpp</exec_depend> -->
<!-- Use build_depend for packages you need at compile time: -->
<build_depend>message_generation</build_depend>
<!-- Use build_export_depend for packages you need in order to build against this package: -->
<!-- <build_export_depend>message_generation</build_export_depend> -->
<!-- Use buildtool_depend for build tool packages: -->
<!-- <buildtool_depend>catkin</buildtool_depend> -->
<!-- Use exec_depend for packages you need at runtime: -->
<exec_depend>message_runtime</exec_depend>
<!-- Use test_depend for packages you need only for testing: -->
<!-- <test_depend>gtest</test_depend> -->
<!-- Use doc_depend for packages you need only for building documentation: -->
<!-- <doc_depend>doxygen</doc_depend> -->
<buildtool_depend>catkin</buildtool_depend>
<build_depend>roscpp</build_depend>
<build_depend>rospy</build_depend>
<build_depend>std_msgs</build_depend>
<build_depend>vision_msgs</build_depend>
<build_export_depend>roscpp</build_export_depend>
<build_export_depend>rospy</build_export_depend>
<build_export_depend>std_msgs</build_export_depend>
<build_export_depend>vision_msgs</build_export_depend>
<exec_depend>roscpp</exec_depend>
<exec_depend>rospy</exec_depend>
<exec_depend>std_msgs</exec_depend>
<exec_depend>vision_msgs</exec_depend>
<!-- The export tag contains other, unspecified, tags -->
<export>
<!-- Other tools can request additional information be placed here -->
</export>
</package>
================================================
FILE: paper_experiments/models/__init__.py
================================================
================================================
FILE: paper_experiments/models/aligned_reid_model.py
================================================
import torch
import torch.nn as nn
import torch.nn.init as init
import torch.nn.functional as F
import torch.utils.model_zoo as model_zoo
import os
import math
class Model(nn.Module):
def __init__(self, local_conv_out_channels=128, num_classes=None):
super(Model, self).__init__()
self.base = resnet50(pretrained=True)
planes = 2048
self.local_conv = nn.Conv2d(planes, local_conv_out_channels, 1)
self.local_bn = nn.BatchNorm2d(local_conv_out_channels)
self.local_relu = nn.ReLU(inplace=True)
if num_classes is not None:
self.fc = nn.Linear(planes, num_classes)
init.normal(self.fc.weight, std=0.001)
init.constant(self.fc.bias, 0)
def forward(self, x):
"""
Returns:
global_feat: shape [N, C]
local_feat: shape [N, H, c]
"""
# shape [N, C, H, W]
feat = self.base(x)
global_feat = F.avg_pool2d(feat, feat.size()[2:])
# shape [N, C]
global_feat = global_feat.view(global_feat.size(0), -1)
# shape [N, C, H, 1]
local_feat = torch.mean(feat, -1, keepdim=True)
local_feat = self.local_relu(self.local_bn(self.local_conv(local_feat)))
# shape [N, H, c]
local_feat = local_feat.squeeze(-1).permute(0, 2, 1)
if hasattr(self, 'fc'):
logits = self.fc(global_feat)
return global_feat, local_feat, logits
return global_feat, local_feat
__all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101',
'resnet152']
model_urls = {
'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',
'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',
'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',
'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',
'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',
}
os.environ["TORCH_HOME"] = "./ResNet_Model"
def conv3x3(in_planes, out_planes, stride=1):
"""3x3 convolution with padding"""
return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
padding=1, bias=False)
class BasicBlock(nn.Module):
expansion = 1
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(BasicBlock, self).__init__()
self.conv1 = conv3x3(inplanes, planes, stride)
self.bn1 = nn.BatchNorm2d(planes)
self.relu = nn.ReLU(inplace=True)
self.conv2 = conv3x3(planes, planes)
self.bn2 = nn.BatchNorm2d(planes)
self.downsample = downsample
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
out = self.bn1(out)
out = self.relu(out)
out = self.conv2(out)
out = self.bn2(out)
if self.downsample is not None:
residual = self.downsample(x)
out += residual
out = self.relu(out)
return out
class Bottleneck(nn.Module):
expansion = 4
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(Bottleneck, self).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
self.bn1 = nn.BatchNorm2d(planes)
self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,
padding=1, bias=False)
self.bn2 = nn.BatchNorm2d(planes)
self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
self.bn3 = nn.BatchNorm2d(planes * 4)
self.relu = nn.ReLU(inplace=True)
self.downsample = downsample
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
out = self.bn1(out)
out = self.relu(out)
out = self.conv2(out)
out = self.bn2(out)
out = self.relu(out)
out = self.conv3(out)
out = self.bn3(out)
if self.downsample is not None:
residual = self.downsample(x)
out += residual
out = self.relu(out)
return out
class ResNet(nn.Module):
def __init__(self, block, layers):
self.inplanes = 64
super(ResNet, self).__init__()
self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,
bias=False)
self.bn1 = nn.BatchNorm2d(64)
self.relu = nn.ReLU(inplace=True)
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.layer1 = self._make_layer(block, 64, layers[0])
self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
def _make_layer(self, block, planes, blocks, stride=1):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=1, stride=stride, bias=False),
nn.BatchNorm2d(planes * block.expansion),
)
layers = []
layers.append(block(self.inplanes, planes, stride, downsample))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes))
return nn.Sequential(*layers)
def forward(self, x):
x = self.conv1(x)
x = self.bn1(x)
x = self.relu(x)
x = self.maxpool(x)
x = self.layer1(x)
x = self.layer2(x)
x = self.layer3(x)
x = self.layer4(x)
return x
def remove_fc(state_dict):
"""Remove the fc layer parameters from state_dict."""
new_state_dict = state_dict.copy()
for key, value in state_dict.items():
if key.startswith('fc.'):
del new_state_dict[key]
return new_state_dict
def resnet18(pretrained=False):
"""Constructs a ResNet-18 model.
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet
"""
model = ResNet(BasicBlock, [2, 2, 2, 2])
if pretrained:
model.load_state_dict(remove_fc(model_zoo.load_url(model_urls['resnet18'])))
return model
def resnet34(pretrained=False):
"""Constructs a ResNet-34 model.
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet
"""
model = ResNet(BasicBlock, [3, 4, 6, 3])
if pretrained:
model.load_state_dict(remove_fc(model_zoo.load_url(model_urls['resnet34'])))
return model
def resnet50(pretrained=False):
"""Constructs a ResNet-50 model.
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet
"""
model = ResNet(Bottleneck, [3, 4, 6, 3])
if pretrained:
model.load_state_dict(remove_fc(model_zoo.load_url(model_urls['resnet50'], model_dir="./ResNet_Model")))### ADDED MODEL_DIR
return model
def resnet101(pretrained=False):
"""Constructs a ResNet-101 model.
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet
"""
model = ResNet(Bottleneck, [3, 4, 23, 3])
if pretrained:
model.load_state_dict(
remove_fc(model_zoo.load_url(model_urls['resnet101'])))
return model
def resnet152(pretrained=False):
"""Constructs a ResNet-152 model.
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet
"""
model = ResNet(Bottleneck, [3, 8, 36, 3])
if pretrained:
model.load_state_dict(
remove_fc(model_zoo.load_url(model_urls['resnet152'])))
return model
================================================
FILE: paper_experiments/models/combination_model.py
================================================
import pdb
import numpy as np
import torch.nn as nn
class CombiNet(nn.Module):
def __init__(self, in_dim = 2560, hidden_units = 512, out_dim = 2560):
super().__init__()
self.fc1 = nn.Linear(in_dim, 2*hidden_units)
# self.bn1 = nn.BatchNorm1d(hidden_units)
self.fc2 = nn.Linear(2*hidden_units, 2*hidden_units)
# self.bn2 = nn.BatchNorm1d(2*hidden_units)
self.fc3 = nn.Linear(2*hidden_units, out_dim)
self.relu = nn.ReLU()
self.apply(weight_init)
def forward(self, x):
# out = nn.functional.normalize(x)
skip = x
out = self.fc1(x)
# out = self.bn1(out)
out = self.relu(out)
out = self.fc2(out)
# out = self.bn2(out)
out = self.relu(out)
out = self.fc3(out)
# out = nn.functional.normalize(out)
out += skip
return out
class CombiLSTM(nn.Module):
def __init__(self, in_dim = 2560, hidden_units = 512, out_dim = 2560):
super().__init__()
self.in_linear1 = nn.Linear(in_dim, hidden_units)
# self.bn1 = nn.BatchNorm1d(hidden_units)
self.in_linear2 = nn.Linear(hidden_units, hidden_units)
self.rnn = nn.LSTM(input_size = hidden_units, hidden_size = hidden_units, dropout = 0)
self.out_linear1 = nn.Linear(hidden_units, hidden_units)
# self.bn2 = nn.BatchNorm1d(hidden_units)
self.out_linear2 = nn.Linear(hidden_units, out_dim)
self.relu = nn.ReLU()
self.apply(weight_init)
def forward(self, x, hidden = None):
out = nn.functional.normalize(x)
skip = out
out = self.in_linear1(out)
# out = self.bn1(out)
out = self.relu(out)
out = self.in_linear2(out)
out = out.unsqueeze(1) #Adding batch dimension
if hidden is None:
out, hidden = self.rnn(out)
else:
out, hidden = self.rnn(out, hidden)
out = out.squeeze(1) #removing batch dimension
out = self.out_linear1(out)
# out = self.bn2(out)
out = self.relu(out)
out = self.out_linear2(out)
out = nn.functional.normalize(out)
out += skip
return out, hidden
def weight_init(m):
if type(m)==nn.Linear:
nn.init.xavier_normal_(m.weight, gain=np.sqrt(2))
elif type(m)==nn.LSTM:
nn.init.xavier_normal_(m.weight_ih_l0)
nn.init.xavier_normal_(m.weight_hh_l0)
================================================
FILE: paper_experiments/models/deep_sort_model.py
================================================
import tensorflow as tf
from skimage.transform import resize
import numpy as np
class ImageEncoder(object):
def __init__(self, checkpoint_filename="weights/deep_sort_weights.pb", input_name="images",
output_name="features"):
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.session = tf.Session(config=config)
with tf.gfile.GFile(checkpoint_filename, "rb") as file_handle:
graph_def = tf.GraphDef()
graph_def.ParseFromString(file_handle.read())
tf.import_graph_def(graph_def, name="net")
self.input_var = tf.get_default_graph().get_tensor_by_name(
"net/%s:0" % input_name)
self.output_var = tf.get_default_graph().get_tensor_by_name(
"net/%s:0" % output_name)
assert len(self.output_var.get_shape()) == 2
assert len(self.input_var.get_shape()) == 4
self.feature_dim = self.output_var.get_shape().as_list()[-1]
self.image_shape = self.input_var.get_shape().as_list()[1:]
def __call__(self, data_x):
#Resize input to expected size for model
data_x = resize(data_x[0], self.image_shape, anti_aliasing=True, mode='reflect')
data_x = np.expand_dims(data_x, 0)
out = self.session.run(self.output_var, feed_dict={self.input_var: data_x})
return out
if __name__ == '__main__':
encoder = ImageEncoder()
================================================
FILE: paper_experiments/models/featurepointnet_model.py
================================================
import os, pdb
import numpy as np
import tensorflow as tf
tf.logging.set_verbosity(tf.logging.ERROR)
import configparser
import utils.featurepointnet_tf_util as tf_util
import utils.featurepointnet_model_util as model_util
from utils.calibration import Calibration, OmniCalibration
batch_size = 45 #TODO: Update if needed?
class FPointNet():
def __init__(self, config_path):
parser = configparser.SafeConfigParser()
parser.read(config_path)
self.num_point = parser.getint('general', 'num_point')
self.model_path = parser.get('general', 'model_path')
with tf.device('/gpu:'+str('0')):
pointclouds_pl, one_hot_vec_pl, labels_pl, centers_pl, \
heading_class_label_pl, heading_residual_label_pl, \
size_class_label_pl, size_residual_label_pl = model_util.placeholder_inputs(batch_size, self.num_point)
is_training_pl = tf.placeholder(tf.bool, shape=())
end_points, depth_feature = self.get_model(pointclouds_pl, one_hot_vec_pl, is_training_pl)
self.object_pointcloud = tf.placeholder(tf.float32, shape=(None, None, 3))
#depth_feature = self.get_depth_feature_op(is_training_pl)
loss = model_util.get_loss(labels_pl, centers_pl, heading_class_label_pl, heading_residual_label_pl, size_class_label_pl, size_residual_label_pl, end_points)
self.saver = tf.train.Saver()
# Create a session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
self.sess = tf.Session(config=config)
#Initialize variables
self.sess.run(tf.global_variables_initializer())
# Restore variables from disk.
self.saver.restore(self.sess, self.model_path)
self.ops = {'pointclouds_pl': pointclouds_pl,
'one_hot_vec_pl': one_hot_vec_pl,
'labels_pl': labels_pl,
'centers_pl': centers_pl,
'heading_class_label_pl': heading_class_label_pl,
'heading_residual_label_pl': heading_residual_label_pl,
'size_class_label_pl': size_class_label_pl,
'size_residual_label_pl': size_residual_label_pl,
'is_training_pl': is_training_pl,
'logits': end_points['mask_logits'],
'center': end_points['center'],
'end_points': end_points,
'depth_feature':depth_feature,
'loss': loss}
# @profile
def __call__(self, input_point_cloud, rot_angle, peds=False):
'''
one_hot_vec = np.zeros((batch_size, 3))
feed_dict = {self.pointclouds_pl: input_point_cloud,
self.one_hot_vec_pl: one_hot_vec,
self.is_training_pl: False}
features = self.sess.run(self.feature,feed_dict=feed_dict)
return features '''
''' Run inference for frustum pointnets in batch mode '''
one_hot_vec = np.zeros((batch_size,3))
if peds:
one_hot_vec[:, 1] = 1
num_batches = input_point_cloud.shape[0]//batch_size + 1
num_inputs = input_point_cloud.shape[0]
if input_point_cloud.shape[0]%batch_size !=0:
input_point_cloud = np.vstack([input_point_cloud, np.zeros((batch_size - input_point_cloud.shape[0]%batch_size, self.num_point, 4))])
else:
num_batches -= 1
logits = np.zeros((input_point_cloud.shape[0], input_point_cloud.shape[1], 2))
centers = np.zeros((input_point_cloud.shape[0], 3))
heading_logits = np.zeros((input_point_cloud.shape[0], model_util.NUM_HEADING_BIN))
heading_residuals = np.zeros((input_point_cloud.shape[0], model_util.NUM_HEADING_BIN))
size_logits = np.zeros((input_point_cloud.shape[0], model_util.NUM_SIZE_CLUSTER))
size_residuals = np.zeros((input_point_cloud.shape[0], model_util.NUM_SIZE_CLUSTER, 3))
mask_mean_prob = np.zeros((input_point_cloud.shape[0],)) # Step scores
heading_prob = np.zeros((input_point_cloud.shape[0],))
size_prob = np.zeros((input_point_cloud.shape[0],))
scores = np.zeros((input_point_cloud.shape[0],)) # 3D box score
features = np.zeros((input_point_cloud.shape[0], 512))
for i in range(num_batches):
ep = self.ops['end_points']
feed_dict = {\
self.ops['pointclouds_pl']: input_point_cloud[i*batch_size: (i+1)*batch_size],
self.ops['one_hot_vec_pl']: one_hot_vec,
self.ops['is_training_pl']: False}
batch_logits, batch_centers, \
batch_heading_scores, batch_heading_residuals, \
batch_size_scores, batch_size_residuals, batch_features = \
self.sess.run([self.ops['logits'], self.ops['center'],
ep['heading_scores'], ep['heading_residuals'],
ep['size_scores'], ep['size_residuals'], self.ops['depth_feature']],
feed_dict=feed_dict)
logits[i*batch_size: (i+1)*batch_size] = batch_logits
centers[i*batch_size: (i+1)*batch_size] = batch_centers
heading_logits[i*batch_size: (i+1)*batch_size] = batch_heading_scores
heading_residuals[i*batch_size: (i+1)*batch_size] = batch_heading_residuals
size_logits[i*batch_size: (i+1)*batch_size] = batch_size_scores
size_residuals[i*batch_size: (i+1)*batch_size] = batch_size_residuals
features[i*batch_size: (i+1)*batch_size] = batch_features[:,0,:]
heading_cls = np.argmax(heading_logits, 1) # B
size_cls = np.argmax(size_logits, 1) # B
heading_res = np.vstack([heading_residuals[i, heading_cls[i]] for i in range(heading_cls.shape[0])])
size_res = np.vstack([size_residuals[i, size_cls[i], :] for i in range(size_cls.shape[0])])
#TODO: Make this accept batches if wanted
boxes = []
for i in range(num_inputs):
box = np.array(model_util.from_prediction_to_label_format(centers[i], heading_cls[i], heading_res[i], size_cls[i], size_res[i], rot_angle[i]))
box[6] = np.squeeze(box[6])
swp = box[5]
box[5] = box[4]
box[4] = swp
boxes.append(box)
boxes = np.vstack(boxes)
return boxes, mask_mean_prob[:num_inputs], features[:num_inputs]
def get_instance_seg_v1_net(self, point_cloud, one_hot_vec, is_training, bn_decay, end_points):
''' 3D instance segmentation PointNet v1 network.
Input:
point_cloud: TF tensor in shape (B,N,4)
frustum point clouds with XYZ and intensity in point channels
XYZs are in frustum coordinate
one_hot_vec: TF tensor in shape (B,3)
length-3 vectors indicating predicted object type
is_training: TF boolean scalar
bn_decay: TF float scalar
end_points: dict
Output:
logits: TF tensor in shape (B,N,2), scores for bkg/clutter and object
end_points: dict
'''
num_point = point_cloud.get_shape()[1].value
net = tf.expand_dims(point_cloud, 2)
net = tf_util.conv2d(net, 64, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv1', bn_decay=bn_decay)
net = tf_util.conv2d(net, 64, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv2', bn_decay=bn_decay)
point_feat = tf_util.conv2d(net, 64, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv3', bn_decay=bn_decay)
net = tf_util.conv2d(point_feat, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv4', bn_decay=bn_decay)
net = tf_util.conv2d(net, 1024, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv5', bn_decay=bn_decay)
global_feat = tf_util.max_pool2d(net, [num_point,1],
padding='VALID', scope='maxpool')
global_feat = tf.concat([global_feat, tf.expand_dims(tf.expand_dims(one_hot_vec, 1), 1)], axis=3)
global_feat_expand = tf.tile(global_feat, [1, num_point, 1, 1])
concat_feat = tf.concat(axis=3, values=[point_feat, global_feat_expand])
net = tf_util.conv2d(concat_feat, 512, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv6', bn_decay=bn_decay)
net = tf_util.conv2d(net, 256, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv7', bn_decay=bn_decay)
net = tf_util.conv2d(net, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv8', bn_decay=bn_decay)
net = tf_util.conv2d(net, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv9', bn_decay=bn_decay)
net = tf_util.dropout(net, is_training, 'dp1', keep_prob=0.5)
logits = tf_util.conv2d(net, 2, [1,1],
padding='VALID', stride=[1,1], activation_fn=None,
scope='conv10')
logits = tf.squeeze(logits, [2]) # BxNxC
return logits, end_points
def get_3d_box_estimation_v1_net(self, object_point_cloud, one_hot_vec,is_training, bn_decay, end_points):
''' 3D Box Estimation PointNet v1 network.
Input:
object_point_cloud: TF tensor in shape (B,M,C)
point clouds in object coordinate
one_hot_vec: TF tensor in shape (B,3)
length-3 vectors indicating predicted object type
Output:
output: TF tensor in shape (B,3+NUM_HEADING_BIN*2+NUM_SIZE_CLUSTER*4)
including box centers, heading bin class scores and residuals,
and size cluster scores and residuals
'''
num_point = object_point_cloud.get_shape()[1].value
net = tf.expand_dims(object_point_cloud, 2)
net = tf_util.conv2d(net, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv-reg1', bn_decay=bn_decay)
net = tf_util.conv2d(net, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv-reg2', bn_decay=bn_decay)
net = tf_util.conv2d(net, 256, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv-reg3', bn_decay=bn_decay)
net = tf_util.conv2d(net, 512, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv-reg4', bn_decay=bn_decay)
features = tf.reduce_max(net, axis = 1)
net = tf_util.max_pool2d(net, [num_point,1],
padding='VALID', scope='maxpool2')
net = tf.squeeze(net, axis=[1,2])
net = tf.concat([net, one_hot_vec], axis=1)
net = tf_util.fully_connected(net, 512, scope='fc1', bn=True,
is_training=is_training, bn_decay=bn_decay)
net = tf_util.fully_connected(net, 256, scope='fc2', bn=True,
is_training=is_training, bn_decay=bn_decay)
# The first 3 numbers: box center coordinates (cx,cy,cz),
# the next NUM_HEADING_BIN*2: heading bin class scores and bin residuals
# next NUM_SIZE_CLUSTER*4: box cluster scores and residuals
output = tf_util.fully_connected(net,
3+model_util.NUM_HEADING_BIN*2+model_util.NUM_SIZE_CLUSTER*4, activation_fn=None, scope='fc3')
return output, end_points, features
def get_model(self, point_cloud, one_hot_vec, is_training, bn_decay=None):
''' Frustum PointNets model. The model predict 3D object masks and
amodel bounding boxes for objects in frustum point clouds.
Input:
point_cloud: TF tensor in shape (B,N,4)
frustum point clouds with XYZ and intensity in point channels
XYZs are in frustum coordinate
one_hot_vec: TF tensor in shape (B,3)
length-3 vectors indicating predicted object type
is_training: TF boolean scalar
bn_decay: TF float scalar
Output:
end_points: dict (map from name strings to TF tensors)
'''
end_points = {}
# 3D Instance Segmentation PointNet
logits, end_points = self.get_instance_seg_v1_net(\
point_cloud, one_hot_vec,
is_training, bn_decay, end_points)
end_points['mask_logits'] = logits
# Masking
# select masked points and translate to masked points' centroid
object_point_cloud_xyz, mask_xyz_mean, end_points = \
model_util.point_cloud_masking(point_cloud, logits, end_points)
# T-Net and coordinate translation
center_delta, end_points = model_util.get_center_regression_net(\
object_point_cloud_xyz, one_hot_vec,
is_training, bn_decay, end_points)
stage1_center = center_delta + mask_xyz_mean # Bx3
end_points['stage1_center'] = stage1_center
# Get object point cloud in object coordinate
object_point_cloud_xyz_new = \
object_point_cloud_xyz - tf.expand_dims(center_delta, 1)
# Amodel Box Estimation PointNet
output, end_points, features = self.get_3d_box_estimation_v1_net(\
object_point_cloud_xyz_new, one_hot_vec,
is_training, bn_decay, end_points)
# Parse output to 3D box parameters
end_points = model_util.parse_output_to_tensors(output, end_points)
end_points['center'] = end_points['center_boxnet'] + stage1_center # Bx3
return end_points, features
def get_depth_feature_op(self, is_training):
net = tf.expand_dims(self.object_pointcloud, 2)
net = tf_util.conv2d(net, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv-reg1', bn_decay=None)
net = tf_util.conv2d(net, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv-reg2', bn_decay=None)
net = tf_util.conv2d(net, 256, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv-reg3', bn_decay=None)
net = tf_util.conv2d(net, 512, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv-reg4', bn_decay=None)
net = tf.reduce_max(net, axis = 1)
return net
def get_depth_feature(self, object_pointcloud):
feed_dict = {self.object_pointcloud:object_pointcloud, self.ops['is_training_pl']:False}
depth_feature = self.sess.run([self.ops['depth_feature']], feed_dict = feed_dict)
return depth_feature
def softmax(self, x):
''' Numpy function for softmax'''
shape = x.shape
probs = np.exp(x - np.max(x, axis=len(shape)-1, keepdims=True))
probs /= np.sum(probs, axis=len(shape)-1, keepdims=True)
return probs
def create_depth_model(model, config_path):
#Note that folder path must be the folder containing the config.yaml file if omni_camera is True
if model == 'FPointNet':
return FPointNet(config_path)
elif model == 'PointNet':
return PointNet(config_path)
================================================
FILE: paper_experiments/models/pointnet_model.py
================================================
import os, pdb
import tensorflow as tf
tf.logging.set_verbosity(tf.logging.ERROR)
import configparser
from utils.pointnet_transform_nets import input_transform_net, feature_transform_net
import utils.pointnet_tf_util as pointnet_tf_util
class PointNet():
def __init__(self, config_path):
parser = configparser.SafeConfigParser()
parser.read(config_path)
num_points = parser.getint('general', 'num_point')
depth_model_path = parser.get('general', 'depth_model_path')
with tf.device('/gpu:'+str(0)):
self.pointclouds_pl, _ = self.placeholder_inputs(1, num_points)
self.is_training_pl = tf.placeholder(tf.bool, shape=())
# simple model
feature = self.get_model(self.pointclouds_pl, self.is_training_pl)
self.feature = feature
# Add ops to save and restore all the variables.
self.saver = tf.train.Saver()
#Create session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
config.log_device_placement = False
self.sess = tf.Session(config=config)
#Initialize variables
self.sess.run(tf.global_variables_initializer())
#Restore model weights
self.saver.restore(self.sess, depth_model_path)
def __call__(self, input_point_cloud):
feed_dict = {self.pointclouds_pl: input_point_cloud,
self.is_training_pl: False}
features = self.sess.run(self.feature,feed_dict=feed_dict)
return features
def placeholder_inputs(self, batch_size, num_point):
pointclouds_pl = tf.placeholder(tf.float32, shape=(batch_size, None, 3))
labels_pl = tf.placeholder(tf.int32, shape=(batch_size))
return pointclouds_pl, labels_pl
def get_model(self, point_cloud, is_training, bn_decay=None):
""" Classification PointNet, input is BxNx3, output Bx40 """
batch_size = point_cloud.get_shape()[0].value
end_points = {}
with tf.variable_scope('transform_net1', reuse=tf.AUTO_REUSE) as sc:
transform = input_transform_net(point_cloud, is_training, bn_decay, K=3)
point_cloud_transformed = tf.matmul(point_cloud, transform)
input_image = tf.expand_dims(point_cloud_transformed, -1)
net = pointnet_tf_util.conv2d(input_image, 64, [1,3],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv1', bn_decay=bn_decay)
net = pointnet_tf_util.conv2d(net, 64, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv2', bn_decay=bn_decay)
with tf.variable_scope('transform_net2', reuse=tf.AUTO_REUSE) as sc:
transform = feature_transform_net(net, is_training, bn_decay, K=64)
end_points['transform'] = transform
net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform)
net_transformed = tf.expand_dims(net_transformed, [2])
net = pointnet_tf_util.conv2d(net_transformed, 64, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv3', bn_decay=bn_decay)
net = pointnet_tf_util.conv2d(net, 128, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv4', bn_decay=bn_decay)
net = pointnet_tf_util.conv2d(net, 1024, [1,1],
padding='VALID', stride=[1,1],
bn=True, is_training=is_training,
scope='conv5', bn_decay=bn_decay)
# Symmetric function: max pooling
net = tf.reduce_max(net, axis = 1)
net = tf.reshape(net, [batch_size, -1])
feature = net
return feature
def get_loss(self, pred, label, end_points, reg_weight=0.001):
""" pred: B*NUM_CLASSES,
label: B, """
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=pred, labels=label)
classify_loss = tf.reduce_mean(loss)
tf.summary.scalar('classify loss', classify_loss)
# Enforce the transformation as orthogonal matrix
transform = end_points['transform'] # BxKxK
K = transform.get_shape()[1].value
mat_diff = tf.matmul(transform, tf.transpose(transform, perm=[0,2,1]))
mat_diff = mat_diff - tf.constant(np.eye(K), dtype=tf.float32)
mat_diff_loss = tf.nn.l2_loss(mat_diff)
tf.summary.scalar('mat loss', mat_diff_loss)
return classify_loss + mat_diff_loss * reg_weight
================================================
FILE: paper_experiments/models/resnet_reid_models.py
================================================
import torch
import torch.nn as nn
from torch.autograd import Variable
import torchvision.models as models
from torchvision import transforms
import torch.nn.functional as F
class FeatureResNet(nn.Module):
def __init__(self,n_layers=50,pretrained=True):
super(FeatureResNet,self).__init__()
if n_layers == 50:
old_model= models.resnet50(pretrained=pretrained)
elif n_layers == 34:
old_model= models.resnet34(pretrained=pretrained)
elif n_layers == 18:
old_model= models.resnet18(pretrained=pretrained)
else:
raise NotImplementedError('resnet%s is not found'%(n_layers))
for name,modules in old_model._modules.items():
if name.find('fc') == -1:
self.add_module(name,modules)
self.output_dim = old_model.fc.in_features
self.pretrained = pretrained
def forward(self,x):
for name,module in self._modules.items():
x = nn.parallel.data_parallel(module, x)
return x.view(x.size(0), -1)
class ResNet(nn.Module):
def __init__(self,n_id,n_layers=50,pretrained=True):
super(ResNet,self).__init__()
if n_layers == 50:
old_model= models.resnet50(pretrained=pretrained)
elif n_layers == 34:
old_model= models.resnet34(pretrained=pretrained)
elif n_layers == 18:
old_model= models.resnet18(pretrained=pretrained)
else:
raise NotImplementedError('resnet%s is not found'%(n_layers))
for name,modules in old_model._modules.items():
self.add_module(name,modules)
self.fc = nn.Linear(self.fc.in_features,n_id)
#########
self.pretrained = pretrained
def forward(self,x):
for name,module in self._modules.items():
if name != 'fc':
x = module(x)
out = self.fc(x.view(x.size(0),-1))
return out, x.view(x.size(0), -1)
class NLayersFC(nn.Module):
def __init__(self, in_dim, out_dim, hidden_dim=1, n_layers=0):
super(NLayersFC, self).__init__()
if n_layers == 0:
model = [nn.Linear(in_dim, out_dim)]
else:
model = []
model += [nn.Linear(in_dim, hidden_dim),
nn.ReLU(True)]
for i in range(n_layers-1):
model += [nn.Linear(hidden_dim, hidden_dim),
nn.ReLU(True)]
model += [nn.Linear(hidden_dim, out_dim)]
self.model = nn.Sequential(*model)
def forward(self, x):
return self.model(x)
class ICT_ResNet(nn.Module):
def __init__(self,n_id,n_color,n_type,n_layers=50,pretrained=True):
super(ICT_ResNet,self).__init__()
if n_layers == 50:
old_model= models.resnet50(pretrained=pretrained)
elif n_layers == 34:
old_model= models.resnet34(pretrained=pretrained)
elif n_layers == 18:
old_model= models.resnet18(pretrained=pretrained)
else:
raise NotImplementedError('resnet%s is not found'%(n_layers))
for name,modules in old_model._modules.items():
self.add_module(name,modules)
self.fc = nn.Linear(self.fc.in_features,n_id)
self.fc_c = nn.Linear(self.fc.in_features,n_color)
self.fc_t = nn.Linear(self.fc.in_features,n_type)
#########
self.pretrained = pretrained
def forward(self,x):
for name,module in self._modules.items():
if name.find('fc')==-1:
x = module(x)
x = x.view(x.size(0),-1)
x_i = self.fc(x)
x_c = self.fc_c(x)
x_t = self.fc_t(x)
return x_i,x_c,x_t
class TripletNet(nn.Module):
def __init__(self, net):
super(TripletNet, self).__init__()
self.net = net
def forward(self, x, y, z):
pred_x, feat_x = self.net(x)
pred_y, feat_y = self.net(y)
pred_z, feat_z = self.net(z)
dist_pos = F.pairwise_distance(feat_x, feat_y, 2)
dist_neg = F.pairwise_distance(feat_x, feat_z, 2)
return dist_pos, dist_neg, pred_x, pred_y, pred_z
if __name__ == '__main__':
netM = ICT_ResNet(n_id=1000,n_color=7,n_type=7,n_layers=18,pretrained=True).cuda()
print(netM)
output = netM(Variable(torch.ones(1,3,224,224).cuda()/2.))
print(output[0].size())
print(output[1].size())
print(output[2].size())
================================================
FILE: paper_experiments/models/yolo_models.py
================================================
from __future__ import division
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
import numpy as np
from PIL import Image
from utils.yolo_utils.parse_config import *
from utils.yolo_utils.utils import build_targets
from collections import defaultdict
import matplotlib.pyplot as plt
import matplotlib.patches as patches
def create_modules(module_defs):
"""
Constructs module list of layer blocks from module configuration in module_defs
"""
hyperparams = module_defs.pop(0)
output_filters = [int(hyperparams["channels"])]
module_list = nn.ModuleList()
for i, module_def in enumerate(module_defs):
modules = nn.Sequential()
if module_def["type"] == "convolutional":
bn = int(module_def["batch_normalize"])
filters = int(module_def["filters"])
kernel_size = int(module_def["size"])
pad = (kernel_size - 1) // 2 if int(module_def["pad"]) else 0
modules.add_module(
"conv_%d" % i,
nn.Conv2d(
in_channels=output_filters[-1],
out_channels=filters,
kernel_size=kernel_size,
stride=int(module_def["stride"]),
padding=pad,
bias=not bn,
),
)
if bn:
modules.add_module("batch_norm_%d" % i, nn.BatchNorm2d(filters))
if module_def["activation"] == "leaky":
modules.add_module("leaky_%d" % i, nn.LeakyReLU(0.1))
elif module_def["type"] == "maxpool":
kernel_size = int(module_def["size"])
stride = int(module_def["stride"])
if kernel_size == 2 and stride == 1:
padding = nn.ZeroPad2d((0, 1, 0, 1))
modules.add_module("_debug_padding_%d" % i, padding)
maxpool = nn.MaxPool2d(
kernel_size=int(module_def["size"]),
stride=int(module_def["stride"]),
padding=int((kernel_size - 1) // 2),
)
modules.add_module("maxpool_%d" % i, maxpool)
elif module_def["type"] == "upsample":
upsample = Interpolate(scale_factor=int(module_def["stride"]), mode="nearest")
modules.add_module("upsample_%d" % i, upsample)
elif module_def["type"] == "route":
layers = [int(x) for x in module_def["layers"].split(",")]
filters = sum([output_filters[layer_i] for layer_i in layers])
modules.add_module("route_%d" % i, EmptyLayer())
elif module_def["type"] == "shortcut":
filters = output_filters[int(module_def["from"])]
modules.add_module("shortcut_%d" % i, EmptyLayer())
elif module_def["type"] == "yolo":
anchor_idxs = [int(x) for x in module_def["mask"].split(",")]
# Extract anchors
anchors = [int(x) for x in module_def["anchors"].split(",")]
anchors = [(anchors[i], anchors[i + 1]) for i in range(0, len(anchors), 2)]
anchors = [anchors[i] for i in anchor_idxs]
num_classes = int(module_def["classes"])
img_height = int(hyperparams["height"])
# Define detection layer
yolo_layer = YOLOLayer(anchors, num_classes, img_height)
modules.add_module("yolo_%d" % i, yolo_layer)
# Register module list and number of output filters
module_list.append(modules)
output_filters.append(filters)
return hyperparams, module_list
class EmptyLayer(nn.Module):
"""Placeholder for 'route' and 'shortcut' layers"""
def __init__(self):
super(EmptyLayer, self).__init__()
class Interpolate(nn.Module):
def __init__(self, scale_factor, mode):
super(Interpolate, self).__init__()
self.interp = nn.functional.interpolate
self.scale_factor = scale_factor
self.mode = mode
def forward(self, x):
x = self.interp(x, scale_factor=self.scale_factor, mode=self.mode)
return x
class YOLOLayer(nn.Module):
"""Detection layer"""
def __init__(self, anchors, num_classes, img_dim):
super(YOLOLayer, self).__init__()
self.anchors = anchors
self.num_anchors = len(anchors)
self.num_classes = num_classes
self.bbox_attrs = 5 + num_classes
self.image_dim = img_dim
self.ignore_thres = 0.5
self.lambda_coord = 1
self.mse_loss = nn.MSELoss(reduction = 'elementwise_mean') # Coordinate loss
self.bce_loss = nn.BCELoss(reduction = 'elementwise_mean') # Confidence loss
self.ce_loss = nn.CrossEntropyLoss() # Class loss
def forward(self, x, targets=None):
nA = self.num_anchors
nB = x.size(0)
nG = x.size(2)
stride = self.image_dim / nG
# Tensors for cuda support
FloatTensor = torch.cuda.FloatTensor if x.is_cuda else torch.FloatTensor
LongTensor = torch.cuda.LongTensor if x.is_cuda else torch.LongTensor
ByteTensor = torch.cuda.ByteTensor if x.is_cuda else torch.ByteTensor
prediction = x.view(nB, nA, self.bbox_attrs, nG, nG).permute(0, 1, 3, 4, 2).contiguous()
# Get outputs
x = torch.sigmoid(prediction[..., 0]) # Center x
y = torch.sigmoid(prediction[..., 1]) # Center y
w = prediction[..., 2] # Width
h = prediction[..., 3] # Height
pred_conf = torch.sigmoid(prediction[..., 4]) # Conf
pred_cls = torch.sigmoid(prediction[..., 5:]) # Cls pred.
# Calculate offsets for each grid
grid_x = torch.arange(nG).repeat(nG, 1).view([1, 1, nG, nG]).type(FloatTensor)
grid_y = torch.arange(nG).repeat(nG, 1).t().view([1, 1, nG, nG]).type(FloatTensor)
scaled_anchors = FloatTensor([(a_w / stride, a_h / stride) for a_w, a_h in self.anchors])
anchor_w = scaled_anchors[:, 0:1].view((1, nA, 1, 1))
anchor_h = scaled_anchors[:, 1:2].view((1, nA, 1, 1))
# Add offset and scale with anchors
pred_boxes = FloatTensor(prediction[..., :4].shape)
pred_boxes[..., 0] = x.data + grid_x
pred_boxes[..., 1] = y.data + grid_y
pred_boxes[..., 2] = torch.exp(w.data) * anchor_w
pred_boxes[..., 3] = torch.exp(h.data) * anchor_h
# Training
if targets is not None:
if x.is_cuda:
self.mse_loss = self.mse_loss.cuda()
self.bce_loss = self.bce_loss.cuda()
self.ce_loss = self.ce_loss.cuda()
nGT, nCorrect, mask, conf_mask, tx, ty, tw, th, tconf, tcls = build_targets(
pred_boxes=pred_boxes.cpu().data,
pred_conf=pred_conf.cpu().data,
pred_cls=pred_cls.cpu().data,
target=targets.cpu().data,
anchors=scaled_anchors.cpu().data,
num_anchors=nA,
num_classes=self.num_classes,
grid_size=nG,
ignore_thres=self.ignore_thres,
img_dim=self.image_dim,
)
nProposals = int((pred_conf > 0.5).sum().item())
recall = float(nCorrect / nGT) if nGT else 1
precision = float(nCorrect / nProposals)
# Handle masks
mask = Variable(mask.type(ByteTensor))
conf_mask = Variable(conf_mask.type(ByteTensor))
# Handle target variables
tx = Variable(tx.type(FloatTensor), requires_grad=False)
ty = Variable(ty.type(FloatTensor), requires_grad=False)
tw = Variable(tw.type(FloatTensor), requires_grad=False)
th = Variable(th.type(FloatTensor), requires_grad=False)
tconf = Variable(tconf.type(FloatTensor), requires_grad=False)
tcls = Variable(tcls.type(LongTensor), requires_grad=False)
# Get conf mask where gt and where there is no gt
conf_mask_true = mask
conf_mask_false = conf_mask - mask
# Mask outputs to ignore non-existing objects
loss_x = self.mse_loss(x[mask], tx[mask])
loss_y = self.mse_loss(y[mask], ty[mask])
loss_w = self.mse_loss(w[mask], tw[mask])
loss_h = self.mse_loss(h[mask], th[mask])
loss_conf = self.bce_loss(pred_conf[conf_mask_false], tconf[conf_mask_false]) + self.bce_loss(
pred_conf[conf_mask_true], tconf[conf_mask_true]
)
loss_cls = (1 / nB) * self.ce_loss(pred_cls[mask], torch.argmax(tcls[mask], 1))
loss = loss_x + loss_y + loss_w + loss_h + loss_conf + loss_cls
return (
loss,
loss_x.item(),
loss_y.item(),
loss_w.item(),
loss_h.item(),
loss_conf.item(),
loss_cls.item(),
recall,
precision,
)
else:
# If not in training phase return predictions
output = torch.cat(
(
pred_boxes.view(nB, -1, 4) * stride,
pred_conf.view(nB, -1, 1),
pred_cls.view(nB, -1, self.num_classes),
),
-1,
)
return output
class Darknet(nn.Module):
"""YOLOv3 object detection model"""
def __init__(self, config_path):
super(Darknet, self).__init__()
self.module_defs = parse_model_config(config_path)
self.hyperparams, self.module_list = create_modules(self.module_defs)
self.seen = 0
self.header_info = np.array([0, 0, 0, self.seen, 0])
self.loss_names = ["x", "y", "w", "h", "conf", "cls", "recall", "precision"]
self.load_weights(self.module_defs[-1]['path'])
def forward(self, x, targets=None):
is_training = targets is not None
output = []
self.losses = defaultdict(float)
layer_outputs = []
for i, (module_def, module) in enumerate(zip(self.module_defs, self.module_list)):
if module_def["type"] in ["convolutional", "upsample", "maxpool"]:
x = module(x)
elif module_def["type"] == "route":
layer_i = [int(x) for x in module_def["layers"].split(",")]
x = torch.cat([layer_outputs[i] for i in layer_i], 1)
elif module_def["type"] == "shortcut":
layer_i = int(module_def["from"])
x = layer_outputs[-1] + layer_outputs[layer_i]
elif module_def["type"] == "yolo":
# Train phase: get loss
if is_training:
x, *losses = module[0](x, targets)
for name, loss in zip(self.loss_names, losses):
self.losses[name] += loss
# Test phase: Get detections
else:
x = module(x)
output.append(x)
layer_outputs.append(x)
self.losses["recall"] /= 3
self.losses["precision"] /= 3
return sum(output) if is_training else torch.cat(output, 1)
def load_weights(self, weights_path):
"""Parses and loads the weights stored in 'weights_path'"""
# Open the weights file
fp = open(weights_path, "rb")
header = np.fromfile(fp, dtype=np.int32, count=5) # First five are header values
# Needed to write header when saving weights
self.header_info = header
self.seen = header[3]
weights = np.fromfile(fp, dtype=np.float32) # The rest are weights
fp.close()
ptr = 0
for i, (module_def, module) in enumerate(zip(self.module_defs, self.module_list)):
if module_def["type"] == "convolutional":
conv_layer = module[0]
if module_def["batch_normalize"]:
# Load BN bias, weights, running mean and running variance
bn_layer = module[1]
num_b = bn_layer.bias.numel() # Number of biases
# Bias
bn_b = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(bn_layer.bias)
bn_layer.bias.data.copy_(bn_b)
ptr += num_b
# Weight
bn_w = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(bn_layer.weight)
bn_layer.weight.data.copy_(bn_w)
ptr += num_b
# Running Mean
bn_rm = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(bn_layer.running_mean)
bn_layer.running_mean.data.copy_(bn_rm)
ptr += num_b
# Running Var
bn_rv = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(bn_layer.running_var)
bn_layer.running_var.data.copy_(bn_rv)
ptr += num_b
else:
# Load conv. bias
num_b = conv_layer.bias.numel()
conv_b = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(conv_layer.bias)
conv_layer.bias.data.copy_(conv_b)
ptr += num_b
# Load conv. weights
num_w = conv_layer.weight.numel()
conv_w = torch.from_numpy(weights[ptr : ptr + num_w]).view_as(conv_layer.weight)
conv_layer.weight.data.copy_(conv_w)
ptr += num_w
"""
@:param path - path of the new weights file
@:param cutoff - save layers between 0 and cutoff (cutoff = -1 -> all are saved)
"""
def save_weights(self, path, cutoff=-1):
fp = open(path, "wb")
self.header_info[3] = self.seen
self.header_info.tofile(fp)
# Iterate through layers
for i, (module_def, module) in enumerate(zip(self.module_defs[:cutoff], self.module_list[:cutoff])):
if module_def["type"] == "convolutional":
conv_layer = module[0]
# If batch norm, load bn first
if module_def["batch_normalize"]:
bn_layer = module[1]
bn_layer.bias.data.cpu().numpy().tofile(fp)
bn_layer.weight.data.cpu().numpy().tofile(fp)
bn_layer.running_mean.data.cpu().numpy().tofile(fp)
bn_layer.running_var.data.cpu().numpy().tofile(fp)
# Load conv bias
else:
conv_layer.bias.data.cpu().numpy().tofile(fp)
# Load conv weights
conv_layer.weight.data.cpu().numpy().tofile(fp)
fp.close()
================================================
FILE: paper_experiments/requirements.txt
================================================
absl-py==0.7.0
astor==0.7.1
backcall==0.1.0
bleach==3.3.0
catkin-pkg==0.4.12
certifi==2018.11.29
chardet==3.0.4
cloudpickle==0.7.0
cupy==6.0.0
cycler==0.10.0
Cython==0.29.7
dask==2021.10.0
decorator==4.3.2
defusedxml==0.5.0
docutils==0.14
entrypoints==0.3
fastrlock==0.4
ffmpeg==1.4
gast==0.2.2
grpcio==1.18.0
gurobipy==8.1.0
html5lib==0.9999999
idna==2.8
imageio==2.5.0
imageio-ffmpeg==0.3.0
ipydatawidgets==4.0.0
ipykernel==5.1.0
ipympl==0.2.1
ipython==7.16.3
ipython-genutils==0.2.0
ipyvolume==0.5.1
ipywebrtc==0.4.3
ipywidgets==7.4.2
jedi==0.13.2
Jinja2==2.11.3
jsonschema==2.6.0
jupyter-client==5.2.4
jupyter-core==4.4.0
jupyterlab==1.2.21
jupyterlab-server==0.2.0
kiwisolver==1.0.1
lap==0.4.0
lapjv==1.3.1
line-profiler==2.1.1
llvmlite==0.28.0
Markdown==3.0.1
MarkupSafe==1.1.0
matplotlib==3.0.2
mistune==0.8.4
nbconvert==5.4.1
nbformat==4.4.0
networkx==2.2
notebook==6.4.1
numba==0.43.1
numpy==1.21.0
open3d-python==0.7.0.0
opencv-python==4.2.0.32
pandas==0.24.1
pandocfilters==1.4.2
parso==0.3.3
pexpect==4.6.0
pickleshare==0.7.5
Pillow==9.0.0
pptk==0.1.0
prometheus-client==0.5.0
prompt-toolkit==2.0.8
protobuf==3.6.1
ptyprocess==0.6.0
pycocotools==2.0.0
Pygments==2.7.4
pyparsing==2.3.1
pypcd==0.1.1
python-dateutil==2.8.0
python-lzf==0.2.4
pythreejs==2.0.2
pytz==2018.9
PyWavelets==1.0.1
PyYAML==5.4
pyzmq==17.1.2
requests==2.21.0
rospkg==1.1.9
scikit-image==0.14.2
scikit-learn==0.20.2
scipy==1.2.0
seaborn==0.9.0
Send2Trash==1.5.0
six==1.12.0
sklearn==0.0
tensorboard==1.8.0
tensorboardX==1.6
tensorflow-gpu==2.5.2
termcolor==1.1.0
terminado==0.8.1
testpath==0.4.2
toolz==0.9.0
torch==1.0.1
torchvision==0.2.1
tornado==5.1.1
tqdm==4.30.0
traitlets==4.3.2
traittypes==0.2.1
urllib3==1.26.5
wcwidth==0.1.7
Werkzeug==0.15.3
widgetsnbextension==3.4.2
================================================
FILE: paper_experiments/track.py
================================================
import open3d as o3d
import torch
import argparse
import os, pdb, sys, copy, pickle
import time
import random
import numpy as np
import tensorflow as tf
from torch.utils.data import DataLoader
from tqdm import tqdm
from models.aligned_reid_model import Model as aligned_reid_model
from utils.yolo_utils.utils import non_max_suppression, load_classes
from models.combination_model import CombiNet, CombiLSTM
from utils.dataset import SequenceDataset, STIPDataset, collate_fn
from models.deep_sort_model import ImageEncoder as deep_sort_model
from utils.tracker import Tracker
from utils.tracker_3d import Tracker_3d
from utils.deep_sort_utils import non_max_suppression as deepsort_nms
from utils.visualise import draw_track
from utils.read_detections import read_ground_truth_2d_detections, read_ground_truth_3d_detections
from utils.tracking_utils import create_detector, convert_detections, combine_features
from utils.tracking_utils import non_max_suppression_3D, non_max_suppression_3D_prime
from utils.aligned_reid_utils import generate_features, generate_features_batched, get_image_patches, create_appearance_model
from utils.featurepointnet_model_util import generate_detections_3d, convert_depth_features
from models.featurepointnet_model import create_depth_model
def parse_arguments():
parser = argparse.ArgumentParser()
parser.add_argument('--sequence_folder', type=str, default='data/KITTI/sequences/0001', help='path to image sequence')
parser.add_argument('--output_folder', type=str, default='results', help='output folder')
parser.add_argument('--aligned_reid_ckpt', type=str, default='weights/aligned_reid_market_weights.ckpt', help='path to model config file')
parser.add_argument('--resnet_reid_ckpt', type=str, default='weights/resnet_reid.ckpt', help='path to model config file')
parser.add_argument('--depth_model', type=str, default='FPointNet', help='type of depth model to use')
parser.add_argument('--depth_config_path', type=str, default='config/featurepointnet.cfg', help='path to model config file')
parser.add_argument('--appearance_model', type=str, default='resnet_reid', help='type of appearance model to use aligned_reid or deepsort or resnet_reid')
parser.add_argument('--conf_thres', type=float, default=0.8, help='object confidence threshold')
parser.add_argument('--depth_weight', type=float, default=1, help='weight of depth feature while concatenating')
parser.add_argument('--nms_thresh', type=float, default=0.56, help='iou thresshold for non-maximum suppression')
parser.add_argument('--n_cpu', type=int, default=4, help='number of cpu threads to use during batch generation')
parser.add_argument('--use_cuda', type=bool, default=True, help='whether to use cuda if available')
parser.add_argument('-p', '--point_cloud', action='store_false', help='Use to disable pointcloud')
parser.add_argument('-o', '--optical_flow_initiation', action='store_false', help='Use to enable optical flow based velocity initiation')
parser.add_argument('-q', '--perfect', action='store_true', help='whether to use perfect assignments')
parser.add_argument('-g', '--ground_truth', action='store_true', help='whether to use ground truth detections')
parser.add_argument('-r', '--reference', action='store_false', help='whether to use reference detections')
parser.add_argument('-t', '--track_3d', action='store_true', help='whether to do 3d tracking')
parser.add_argument('--ref_det', type = str, default = 'new_rrc_subcnn_car', help='lsvm, subcnn, regionlets, maskrcnn')
parser.add_argument("--nn_budget", type=int, default=100, help="Maximum size of the appearance descriptors gallery. If None, no budget is enforced.")
parser.add_argument("--dummy_node_cost_app", type=float, default=0.99, help="Dummy node appearance cost for JPDA (or maximum distnce when using deepsort)")
parser.add_argument("--dummy_node_cost_iou", type=float, default=0.97, help="Dummy node iou cost for JPDA (or maximum distnce when using deepsort)")
parser.add_argument("-c", "--combine_features", action = 'store_false', help="Whether to use trained MLP to combine features")
parser.add_argument("-f", "--fpointnet", action = 'store_false', help="Whether to use F-PointNet for 3d detection")
parser.add_argument("--combo_model", default = 'weights/resnet_reid_fpointnet_combo_car/mlp__1570759353.0113978/best_checkpoint.tar"', help="Trained MLP checkpoint to combine features")
parser.add_argument("-j", "--JPDA", action = 'store_false', help="Whether to use JPDA for soft assignments")
parser.add_argument("-l", "--LSTM", action = 'store_true', help="Whether to use LSTM for feature combination and update")
parser.add_argument("--lstm_model", default = 'weights/aligned_reid_fpointnet_combo/lstm/best_checkpoint.tar', help="Trained LSTM checkpoint to combine features")
parser.add_argument("-m","--m_best_sol", type=int, default=10, help="Number of solutions for JPDA")
parser.add_argument("--log_data", action='store_true', help="Turn on full data logging")
parser.add_argument("--max_age", type=int, default=2, help="Number of misses before termination")
parser.add_argument("--n_init", type=int, default=2, help="Consecutive frames for tentative->confirmed")
parser.add_argument("--assn_thresh", type=float, default=0.65, help="min prob for match")
parser.add_argument("--matching_strategy", type=str, default="hungarian", help="matching strategy for JPDA (max_and_threshold, strict_max_pair, or hungarian)")
parser.add_argument("--kf_appearance_feature", type=bool, default=False, help="Whether to use kf state for apperance features")
parser.add_argument('-i', "--use_imm", action = 'store_true', help='Whether to use IMM')
parser.add_argument('-v', "--verbose", action = 'store_true', help='Verbose')
parser.add_argument('--kf_process', type=float, default=5.2, help='kf 2d process noise factor')
parser.add_argument('--kf_2d_meas', type=float, default=3.2, help='kf 2d measurement noise factor')
parser.add_argument('--kf_3d_meas', type=float, default=0.25, help='kf 3d measurement noise factor')
parser.add_argument('--pos_weight_3d', type=float, default=1, help='Weight on position covariance process noise in KF')
parser.add_argument('--pos_weight', type=float, default=0.006, help='Weight on position covariance process noise in KF')
parser.add_argument('--vel_weight', type=float, default=0.008, help='Weight on velocity covariance process noise in KF')
parser.add_argument('--theta_weight', type=float, default=0.02, help='Weight on velocity covariance process noise in KF')
parser.add_argument('--gate_limit', type=float, default=600, help='Maximum covariance value of the gate')
parser.add_argument('--initial_uncertainty', type=float, default=1, help='Uncertainty scaling for initial covariance of track')
parser.add_argument('--uncertainty_limit', type=float, default=1.5, help='Uncertainty limit at which to terminate tracks')
parser.add_argument("--gate_full_state", action='store_true', help="Whether to gate on full kalman state, default is only position")
parser.add_argument("--near_online", action = 'store_true', help="Whether to do near online tracking")
parser.add_argument("--omni", action = 'store_true', help="Omni directional camera (JRDB)")
opt = parser.parse_args()
opt.sequence_folder = opt.sequence_folder.rstrip(os.sep)
opt.using_cuda = torch.cuda.is_available() and opt.use_cuda
if not opt.point_cloud and opt.track_3d:
raise("Must provide point cloud if doing 3D tracking!")
if opt.verbose:
print(opt)
if not os.path.exists(opt.output_folder):
os.makedirs(opt.output_folder)
return opt
# @profile
def main(opt):
if opt.verbose:
print("------------------------")
print("RUNNING SET UP")
print("------------------------")
tf.logging.set_verbosity(40)
random.seed(0)
Tensor = torch.cuda.FloatTensor if opt.using_cuda else torch.FloatTensor
os.makedirs(opt.output_folder, exist_ok=True)
if opt.LSTM:
opt.max_cosine_distance = 1
lstm = CombiLSTM()
checkpoint = torch.load(opt.lstm_model)
lstm.load_state_dict(checkpoint['state_dict'])
if opt.using_cuda:
lstm.cuda()
lstm.eval()
else:
lstm = None
if opt.combine_features:
combination_model = CombiNet()
checkpoint = torch.load(opt.combo_model)
combination_model.load_state_dict(checkpoint['state_dict'])
if opt.using_cuda:
combination_model.cuda()
combination_model.eval()
else:
combination_model = None
dataset = SequenceDataset(opt.sequence_folder, point_cloud=opt.point_cloud, omni=opt.omni)
dataloader = DataLoader(dataset, batch_size=1, shuffle=False, num_workers=opt.n_cpu, collate_fn = collate_fn)
appearance_model = create_appearance_model(opt.appearance_model, opt.aligned_reid_ckpt, opt.resnet_reid_ckpt, opt.using_cuda)
if opt.point_cloud:
depth_model = create_depth_model(opt.depth_model, opt.depth_config_path)
if opt.track_3d:
tracker = Tracker_3d(appearance_model=appearance_model, cuda=opt.using_cuda, JPDA = opt.JPDA, m_best_sol=opt.m_best_sol,
max_age = opt.max_age, n_init=opt.n_init, assn_thresh=opt.assn_thresh,
matching_strategy=opt.matching_strategy,
gate_full_state=opt.gate_full_state,
kf_vel_params=(opt.pos_weight_3d, opt.pos_weight, opt.vel_weight, opt.theta_weight,
opt.kf_process, opt.kf_2d_meas, opt.kf_3d_meas, opt.initial_uncertainty),
calib=dataset.calib,
dummy_node_cost_iou=opt.dummy_node_cost_iou,
dummy_node_cost_app=opt.dummy_node_cost_app,
nn_budget=opt.nn_budget,
use_imm=opt.use_imm,
uncertainty_limit=opt.uncertainty_limit,
gate_limit=opt.gate_limit,
omni=opt.omni)
else:
tracker = Tracker(appearance_model=appearance_model, cuda=opt.using_cuda, JPDA = opt.JPDA, m_best_sol=opt.m_best_sol,
max_age = opt.max_age, n_init=opt.n_init, assn_thresh=opt.assn_thresh,
matching_strategy=opt.matching_strategy,
kf_appearance_feature=opt.kf_appearance_feature,
gate_full_state=opt.gate_full_state,
kf_vel_params=(opt.pos_weight, opt.vel_weight, opt.kf_process, opt.kf_2d_meas, opt.initial_uncertainty),
kf_walk_params=(opt.pos_weight, opt.vel_weight, opt.kf_process, opt.kf_2d_meas, opt.initial_uncertainty),
calib=dataset.calib,
dummy_node_cost_iou=opt.dummy_node_cost_iou,
dummy_node_cost_app=opt.dummy_node_cost_app,
nn_budget=opt.nn_budget,
use_imm=opt.use_imm,
uncertainty_limit=opt.uncertainty_limit,
optical_flow=opt.optical_flow_initiation,
gate_limit=opt.gate_limit)
results = []
results_3d = []
n_frames = len(dataloader)
if opt.log_data:
full_log = [{'tracks':[], 'detections':[], 'detections_3d':[]} for _ in range(n_frames)]
det_matrix = None
seq_name = os.path.split(opt.sequence_folder)[-1]
frame_times = []
if opt.verbose:
print("------------------------")
print("BEGINNING TRACKING OF SEQUENCE %s"%seq_name)
print("------------------------")
for frame_idx, img_path, input_img, point_cloud in tqdm(dataloader, ncols = 100, disable=not opt.verbose):
# if frame_idx > 120:
# break
# elif frame_idx < 98:
# continue
if opt.log_data:
full_log[frame_idx]['img_path'] = copy.copy(img_path)
input_img = input_img.type(Tensor)
if opt.reference:
detections, object_ids, det_matrix = read_ground_truth_2d_detections(os.path.join(opt.sequence_folder,'det',opt.ref_det+'.txt'), frame_idx, det_matrix, threshold = 0, nms_threshold = opt.nms_thresh)
elif opt.ground_truth:
detections, object_ids, det_matrix = read_ground_truth_2d_detections(os.path.join(opt.sequence_folder,'gt','gt.txt'), frame_idx, det_matrix, nms_threshold = opt.nms_thresh)
else:
raise("Must specify ground truth or detections")
# --- START OF TRACKING ---
# start_time = time.time()
if detections is None or len(detections)==0:
tracker.predict()
if opt.log_data:
full_log[frame_idx]['predicted_tracks'] = copy.deepcopy(tracker.tracks)
start_time = time.time()
tracker.update(input_img, [])
else:
total_dets = len(detections)
patches = get_image_patches(input_img, detections)
appearance_features = generate_features_batched(appearance_model, patches, opt, object_ids)
if opt.point_cloud:
if not opt.omni:
point_cloud = point_cloud[point_cloud[:,2]>=0]
if opt.fpointnet:
boxes_3d, valid_3d, _, scores_3d, depth_features = generate_detections_3d(depth_model,
detections, np.asarray(point_cloud),
dataset.calib, input_img.shape,
peds='ped' in opt.ref_det or opt.omni)
depth_features = convert_depth_features(depth_features, valid_3d)
else:
boxes_3d, valid_3d = read_ground_truth_3d_detections(os.path.join(opt.sequence_folder,'gt','3d_detections.txt'), frame_idx)
features, appearance_features = combine_features(appearance_features, depth_features, valid_3d, combination_model, depth_weight = opt.depth_weight)
# boxes_3d = boxes_3d[valid_3d != -1] # Old and buggy way of handling missing box
# detections = detections[valid_3d != -1]
if np.any(valid_3d == -1):
compare_2d = True
else:
compare_2d = False
if len(boxes_3d) > 0:
detections_3d = []
for idx, box in enumerate(boxes_3d):
if valid_3d[idx] == -1:
detections_3d.append(None)
else:
detections_3d.append(np.array(box).astype(np.float32))
else:
detections_3d = None
else:
appearance_features = [appearance_features[i] for i in range(total_dets)]
features = [None]*len(appearance_features)
compare_2d = True
detections_3d = None
detections = convert_detections(detections, features, appearance_features, detections_3d)
tracker.predict()
if opt.log_data:
full_log[frame_idx]['predicted_tracks'] = copy.deepcopy(tracker.tracks)
start_time = time.time()
tracker.update(input_img, detections, compare_2d)
# --- END OF TRACKING ---
end_time = time.time()
frame_times.append(end_time - start_time)
if opt.log_data:
full_tracks = copy.deepcopy(tracker.tracks)
temp_tracks = []
for track in full_tracks:
bbox = track.to_tlwh(None)
if not (bbox[0] < 0-10 or bbox[1] < 0-10 or bbox[0] + bbox[2] > input_img.shape[2]+10 or bbox[1] + bbox[3] > input_img.shape[1]+10):
temp_tracks.append(track)
full_log[frame_idx]['tracks'] = temp_tracks
full_log[frame_idx]['detections'] = copy.deepcopy(detections)
for track in tracker.tracks:
if opt.track_3d:
bbox_3d = track.to_tlwh3d()
else:
bbox = track.to_tlwh(None)
if bbox[0] < 0-10 or bbox[1] < 0-10 or bbox[0] + bbox[2] > input_img.shape[2]+10 or bbox[1] + bbox[3] > input_img.shape[1]+10:
continue
bbox[0] = max(0,bbox[0]) # Frame adjustments
bbox[1] = max(0,bbox[1])
bbox[2] = min(bbox[0]+bbox[2], input_img.shape[2])-bbox[0]
bbox[3] = min(bbox[1]+bbox[3], input_img.shape[1])-bbox[1]
track_status = 1
if not track.is_confirmed(): # or track.time_since_update > 0:
if opt.near_online:
if not track.is_confirmed():
track_status = 0
else:
track_status = 2
continue
else:
continue
if opt.near_online:
if opt.track_3d:
results_3d.append([frame_idx, track.track_id, bbox_3d[0], bbox_3d[1], bbox_3d[2], bbox_3d[3], bbox_3d[4], bbox_3d[5], bbox_3d[6], track_status])
else:
results.append([frame_idx, track.track_id, bbox[0], bbox[1], bbox[2], bbox[3], track_status])
if track_status == 1: #updates 0s
for row_i in range(len(results)):
if results[row_i][1] == track.track_id:
results[row_i][6] = 1
if opt.point_cloud:
if results_3d[row_i][1] == track.track_id:
results_3d[row_i][7] = 1
else:
if opt.track_3d:
results_3d.append([frame_idx, track.track_id, bbox_3d[0], bbox_3d[1], bbox_3d[2], bbox_3d[3], bbox_3d[4], bbox_3d[5], bbox_3d[6]])
else:
results.append([frame_idx, track.track_id, bbox[0], bbox[1], bbox[2], bbox[3]])
# if opt.point_cloud:
frame_times = np.asarray(frame_times)
if opt.verbose:
print("------------------------")
print("COMPLETED TRACKING, SAVING RESULTS")
print("------------------------")
print('\n\n','Total Tracking Time:',np.sum(frame_times),'Average Time Per Frame:',np.mean(frame_times))
if opt.track_3d:
output_file_3d = os.path.join(opt.output_folder, seq_name+"_3d.txt")
if len(results_3d) > 0:
with open(output_file_3d, 'w+') as f:
for row in results_3d:
if opt.near_online and row[9] != 1:
continue
print('%d,%d,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.4f,1,1,1,-1' % (
row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8]), file=f)
else:
output_file = os.path.join(opt.output_folder, seq_name+".txt")
if len(results) > 0:
with open(output_file, 'w+') as f:
for row in results:
if opt.near_online and row[6] != 1:
continue
print('%d,%d,%.2f,%.2f,%.2f,%.2f,1,1,1,-1' % (
row[0], row[1], row[2], row[3], row[4], row[5]), file=f)
if opt.log_data:
output_file = os.path.join(opt.output_folder, seq_name+".p")
with open(output_file, 'wb') as f:
pickle.dump(full_log, f)
if __name__=='__main__':
opt = parse_arguments()
main(opt)
================================================
FILE: paper_experiments/utils/EKF.py
================================================
# vim: expandtab:ts=4:sw=4
import numpy as np
import scipy.linalg
import pdb
"""
Table for the 0.95 quantile of the chi-square distribution with N degrees of
freedom (contains values for N=1, ..., 9). Taken from MATLAB/Octave's chi2inv
function and used as Mahalanobis gating threshold.
"""
chi2inv95 = {
1: 3.8415,
2: 5.9915,
3: 7.8147,
4: 9.4877,
5: 11.070,
6: 12.592,
7: 14.067,
8: 15.507,
9: 16.919}
chi2inv90 = {
1: 2.706,
2: 4.605,
3: 6.251,
4: 7.779,
5: 9.236,
6: 10.645,
7: 12.017,
8: 13.363,
9: 14.684}
chi2inv975 = {
1: 5.025,
2: 7.378,
3: 9.348,
4: 11.143,
5: 12.833,
6: 14.449,
7: 16.013,
8: 17.535,
9: 19.023}
chi2inv10 = {
1: .016,
2: .221,
3: .584,
4: 1.064,
5: 1.610,
6: 2.204,
7: 2.833,
8: 3.490,
9: 4.168}
chi2inv995 = {
1: 0.0000393,
2: 0.0100,
3: .0717,
4: .207,
5: .412,
6: .676,
7: .989,
8: 1.344,
9: 1.735}
chi2inv75 = {
1: 1.323,
2: 2.773,
3: 4.108,
4: 5.385,
5: 6.626,
6: 7.841,
7: 9.037,
8: 10.22,
9: 11.39}
def squared_mahalanobis_distance(mean, covariance, measurements):
# cholesky factorization used to solve for
# z = d * inv(covariance)
# so z is also the solution to
# covariance * z = d
d = measurements - mean
# Note: The cholesky factorization is giving weird answers. This is marginally slower but correct
return np.matmul(np.matmul(d, np.linalg.inv(covariance)), d.T).diagonal()
# print("Measurements:", measurements)
# print("Mean:", mean)
# print("dshape:", d.shape, "d:", d)
# print("d*inv(cov)", np.matmul(d, np.linalg.inv(covariance)))
cholesky_factor = np.linalg.cholesky(covariance)
z = scipy.linalg.solve_triangular(
cholesky_factor, d.T, lower=True, check_finite=False,
overwrite_b=True)
squared_maha = np.sum(z * (measurements-mean).T, axis=0)
# print("Squared maha dist:", squared_maha)
# print("cov:", covariance)
# print("z", z, '\n')
return squared_maha
class EKF(object):
"""
Generic extended kalman filter class
"""
def __init__(self):
pass
def initiate(self, measurement):
"""Create track from unassociated measurement.
Parameters
----------
measurement : ndarray
Returns
-------
(ndarray, ndarray)
Returns the mean vector and covariance matrix of the new track.
Unobserved velocities are initialized to 0 mean.
"""
pass
def predict_mean(self, mean):
# Updates predicted state from previous state (function g)
# Calculates motion update Jacobian (Gt)
# Returns (g(mean), Gt)
pass
def get_process_noise(self, mean, covariance):
# Returns Rt the motion noise covariance
pass
def predict_covariance(self, mean, covariance):
pass
def project_mean(self, mean):
# Measurement prediction from state (function h)
# Calculations sensor update Jacobian (Ht)
# Returns (h(mean), Ht)
pass
def project_cov(self, mean, covariance):
pass
def predict(self, mean, covariance, last_detection, next_to_last_detection):
"""Run Kalman filter prediction step.
Parameters
----------
mean : ndarray
The mean vector of the object state at the previous
time step.
covariance : ndarray
The covariance matrix of the object state at the
previous time step.
Returns
-------
(ndarray, ndarray)
Returns the mean vector and covariance matrix of the predicted
state. Unobserved velocities are initialized to 0 mean.
"""
# Perform prediction
covariance = self.predict_covariance(mean, covariance, last_detection, next_to_last_detection)
mean = self.predict_mean(mean)
return mean, covariance
def get_innovation_cov(self, covariance):
pass
def project(self, mean, covariance):
"""Project state distribution to measurement space.
Parameters
----------
mean : ndarray
The state's mean vector
covariance : ndarray
The state's covariance matrix
Returns
-------
(ndarray, ndarray)
Returns the projected mean and covariance matrix of the given state
estimate.
"""
# Measurement uncertainty scaled by estimated height
return self.project_mean(mean), self.project_cov(mean, covariance)
def update(self, mean, covariance, measurement_t, marginalization=None, JPDA=False):
"""Run Kalman filter correction step.
Parameters
----------
mean : ndarray
The predicted state's mean vector (8 dimensional).
covariance : ndarray
The state's covariance matrix (8x8 dimensional).
measurement : ndarray
The 4 dimensional measurement vector (x, y, a, h), where (x, y)
is the center position, a the aspect ratio, and h the height of the
bounding box.
Returns
-------
(ndarray, ndarray)
Returns the measurement-corrected state distribution.
"""
predicted_measurement, innovation_cov = self.project(mean, covariance)
# cholesky factorization used to solve for kalman gain since
# K = covariance * update_mat.T * inv(innovation_cov)
# so K is also the solution to
# innovation_cov * K = covariance * update_mat.T
try:
chol_factor, lower = scipy.linalg.cho_factor(
innovation_cov, lower=True, check_finite=False)
kalman_gain = scipy.linalg.cho_solve(
(chol_factor, lower), np.dot(covariance, self._observation_mat.T).T,
check_finite=False).T
except:
# in case cholesky factorization fails, revert to standard solver
kalman_gain = np.linalg.solve(innovation_cov, np.dot(covariance, self._observation_mat.T).T).T
if JPDA:
# marginalization
innovation = np.zeros((self.ndim))
cov_soft = np.zeros((self.ndim, self.ndim))
for measurement_idx, measurement in enumerate(measurement_t):
p_ij = marginalization[measurement_idx + 1] # + 1 for dummy
y_ij = measurement - predicted_measurement
innovation += y_ij * p_ij
cov_soft += p_ij * np.outer(y_ij, y_ij)
cov_soft = cov_soft - np.outer(innovation, innovation)
P_star = covariance - np.linalg.multi_dot((
kalman_gain, innovation_cov, kalman_gain.T))
p_0 = marginalization[0]
P_0 = p_0 * covariance + (1 - p_0) * P_star
new_covariance = P_0 + np.linalg.multi_dot((kalman_gain, cov_soft, kalman_gain.T))
else:
innovation = measurement_t - predicted_measurement
new_covariance = covariance - np.linalg.multi_dot((
kalman_gain, innovation_cov, kalman_gain.T))
new_mean = mean + np.dot(innovation, kalman_gain.T)
return new_mean, new_covariance
================================================
FILE: paper_experiments/utils/JPDA_matching.py
================================================
# vim: expandtab:ts=4:sw=4
from __future__ import absolute_import
import numpy as np
from linear_assignment import min_marg_matching
import pdb
def get_unmatched(all_idx, matches, i, marginalization=None):
assigned = [match[i] for match in matches]
unmatched = set(all_idx) - set(assigned)
if marginalization is not None:
# from 1 for dummy node
in_gate_dets = np.nonzero(np.sum(
marginalization[:, 1:], axis=0))[0].tolist()
# unmatched = [d for d in unmatched if d not in in_gate_dets] # TODO: Filter by gate?
return list(unmatched)
class Matcher:
def __init__(self, detections, marginalizations, confirmed_tracks,
matching_strategy,
assignment_threshold=None):
self.detections = detections
self.marginalizations = marginalizations
self.confirmed_tracks = confirmed_tracks
self.assignment_threshold = assignment_threshold
self.detection_indices = np.arange(len(detections))
self.matching_strategy = matching_strategy
def match(self):
self.get_matches()
self.get_unmatched_tracks()
self.get_unmatched_detections()
return self.matches, self.unmatched_tracks, self.unmatched_detections
def get_matches(self):
if self.matching_strategy == "max_and_threshold":
self.max_and_threshold_matching()
elif self.matching_strategy == "hungarian":
self.hungarian()
elif self.matching_strategy == "max_match":
self.max_match()
elif self.matching_strategy == "none":
self.matches = []
else:
raise Exception('Unrecognized matching strategy: {}'.
format(self.matching_strategy))
def get_unmatched_tracks(self):
self.unmatched_tracks = get_unmatched(self.confirmed_tracks,
self.matches, 0)
def get_unmatched_detections(self):
self.unmatched_detections = get_unmatched(self.detection_indices, self.matches, 1, self.marginalizations)
def max_match(self):
self.matches = []
if self.marginalizations.shape[0] == 0:
return
detection_map = {}
for i, track_idx in enumerate(self.confirmed_tracks):
marginalization = self.marginalizations[i,:]
detection_id = np.argmax(marginalization) - 1 # subtract one for dummy
if detection_id < 0:
continue
if detection_id not in detection_map.keys():
detection_map[detection_id] = track_idx
else:
cur_track = detection_map[detection_id]
track_update = track_idx if self.marginalizations[track_idx, detection_id] > self.marginalizations[cur_track, detection_id] else cur_track
detection_map[detection_id] = track_update
threshold_p = marginalization[detection_id + 1]
if threshold_p < self.assignment_threshold:
continue
for detection in detection_map.keys():
self.matches.append((detection_map[detection], detection))
def max_and_threshold_matching(self):
self.matches = []
if self.marginalizations.shape[0] == 0:
return
for i, track_idx in enumerate(self.confirmed_tracks):
marginalization = self.marginalizations[i,:]
detection_id = np.argmax(marginalization) - 1 # subtract one for dummy
if detection_id < 0:
continue
threshold_p = marginalization[detection_id + 1]
if threshold_p < self.assignment_threshold:
continue
self.matches.append((track_idx, detection_id))
def hungarian(self):
self.matches, _, _ = min_marg_matching(self.marginalizations,
self.confirmed_tracks,
self.assignment_threshold)
================================================
FILE: paper_experiments/utils/aligned_reid_utils.py
================================================
from __future__ import print_function
import os
import os.path as osp
import pickle
from scipy import io
import datetime
import time
from contextlib import contextmanager
import numpy as np
from PIL import Image
import torch
from torch.autograd import Variable
from models.aligned_reid_model import Model as aligned_reid_model
from models.deep_sort_model import ImageEncoder as deep_sort_model
from utils.resnet_reid_utils import ResNet_Loader
def time_str(fmt=None):
if fmt is None:
fmt = '%Y-%m-%d_%H:%M:%S'
return datetime.datetime.today().strftime(fmt)
def load_pickle(path):
"""Check and load pickle object.
According to this post: https://stackoverflow.com/a/41733927, cPickle and
disabling garbage collector helps with loading speed."""
assert osp.exists(path)
# gc.disable()
with open(path, 'rb') as f:
ret = pickle.load(f)
# gc.enable()
return ret
def save_pickle(obj, path):
"""Create dir and save file."""
may_make_dir(osp.dirname(osp.abspath(path)))
with open(path, 'wb') as f:
pickle.dump(obj, f, protocol=2)
def save_mat(ndarray, path):
"""Save a numpy ndarray as .mat file."""
io.savemat(path, dict(ndarray=ndarray))
def to_scalar(vt):
"""Transform a length-1 pytorch Variable or Tensor to scalar.
Suppose tx is a torch Tensor with shape tx.size() = torch.Size([1]),
then npx = tx.cpu().numpy() has shape (1,), not 1."""
if isinstance(vt, Variable):
return vt.data.cpu().numpy().flatten()[0]
if torch.is_tensor(vt):
return vt.cpu().numpy().flatten()[0]
raise TypeError('Input should be a variable or tensor')
def transfer_optim_state(state, device_id=-1):
"""Transfer an optimizer.state to cpu or specified gpu, which means
transferring tensors of the optimizer.state to specified device.
The modification is in place for the state.
Args:
state: An torch.optim.Optimizer.state
device_id: gpu id, or -1 which means transferring to cpu
"""
for key, val in state.items():
if isinstance(val, dict):
transfer_optim_state(val, device_id=device_id)
elif isinstance(val, Variable):
raise RuntimeError("Oops, state[{}] is a Variable!".format(key))
elif isinstance(val, torch.nn.Parameter):
raise RuntimeError("Oops, state[{}] is a Parameter!".format(key))
else:
try:
if device_id == -1:
state[key] = val.cpu()
else:
state[key] = val.cuda(device=device_id)
except:
pass
def may_transfer_optims(optims, device_id=-1):
"""Transfer optimizers to cpu or specified gpu, which means transferring
tensors of the optimizer to specified device. The modification is in place
for the optimizers.
Args:
optims: A list, which members are either torch.nn.optimizer or None.
device_id: gpu id, or -1 which means transferring to cpu
"""
for optim in optims:
if isinstance(optim, torch.optim.Optimizer):
transfer_optim_state(optim.state, device_id=device_id)
def may_transfer_modules_optims(modules_and_or_optims, device_id=-1):
"""Transfer optimizers/modules to cpu or specified gpu.
Args:
modules_and_or_optims: A list, which members are either torch.nn.optimizer
or torch.nn.Module or None.
device_id: gpu id, or -1 which means transferring to cpu
"""
for item in modules_and_or_optims:
if isinstance(item, torch.optim.Optimizer):
transfer_optim_state(item.state, device_id=device_id)
elif isinstance(item, torch.nn.Module):
if device_id == -1:
item.cpu()
else:
item.cuda(device=device_id)
elif item is not None:
print('[Warning] Invalid type {}'.format(item.__class__.__name__))
class TransferVarTensor(object):
"""Return a copy of the input Variable or Tensor on specified device."""
def __init__(self, device_id=-1):
self.device_id = device_id
def __call__(self, var_or_tensor):
return var_or_tensor.cpu() if self.device_id == -1 \
else var_or_tensor.cuda(self.device_id)
class TransferModulesOptims(object):
"""Transfer optimizers/modules to cpu or specified gpu."""
def __init__(self, device_id=-1):
self.device_id = device_id
def __call__(self, modules_and_or_optims):
may_transfer_modules_optims(modules_and_or_optims, self.device_id)
def set_devices(sys_device_ids):
"""
It sets some GPUs to be visible and returns some wrappers to transferring
Variables/Tensors and Modules/Optimizers.
Args:
sys_device_ids: a tuple; which GPUs to use
e.g. sys_device_ids = (), only use cpu
sys_device_ids = (3,), use the 4th gpu
sys_device_ids = (0, 1, 2, 3,), use first 4 gpus
sys_device_ids = (0, 2, 4,), use the 1st, 3rd and 5th gpus
Returns:
TVT: a `TransferVarTensor` callable
TMO: a `TransferModulesOptims` callable
"""
# Set the CUDA_VISIBLE_DEVICES environment variable
import os
visible_devices = ''
for i in sys_device_ids:
visible_devices += '{}, '.format(i)
os.environ['CUDA_VISIBLE_DEVICES'] = visible_devices
# Return wrappers.
# Models and user defined Variables/Tensors would be transferred to the
# first device.
device_id = 0 if len(sys_device_ids) > 0 else -1
TVT = TransferVarTensor(device_id)
TMO = TransferModulesOptims(device_id)
return TVT, TMO
def set_devices_for_ml(sys_device_ids):
"""This version is for mutual learning.
It sets some GPUs to be visible and returns some wrappers to transferring
Variables/Tensors and Modules/Optimizers.
Args:
sys_device_ids: a tuple of tuples; which devices to use for each model,
len(sys_device_ids) should be equal to number of models. Examples:
sys_device_ids = ((-1,), (-1,))
the two models both on CPU
sys_device_ids = ((-1,), (2,))
the 1st model on CPU, the 2nd model on GPU 2
sys_device_ids = ((3,),)
the only one model on the 4th gpu
sys_device_ids = ((0, 1), (2, 3))
the 1st model on GPU 0 and 1, the 2nd model on GPU 2 and 3
sys_device_ids = ((0,), (0,))
the two models both on GPU 0
sys_device_ids = ((0,), (0,), (1,), (1,))
the 1st and 2nd model on GPU 0, the 3rd and 4th model on GPU 1
Returns:
TVTs: a list of `TransferVarTensor` callables, one for one model.
TMOs: a list of `TransferModulesOptims` callables, one for one model.
relative_device_ids: a list of lists; `sys_device_ids` transformed to
relative ids; to be used in `DataParallel`
"""
import os
all_ids = []
for ids in sys_device_ids:
all_ids += ids
unique_sys_device_ids = list(set(all_ids))
unique_sys_device_ids.sort()
if -1 in unique_sys_device_ids:
unique_sys_device_ids.remove(-1)
# Set the CUDA_VISIBLE_DEVICES environment variable
visible_devices = ''
for i in unique_sys_device_ids:
visible_devices += '{}, '.format(i)
os.environ['CUDA_VISIBLE_DEVICES'] = visible_devices
# Return wrappers
relative_device_ids = []
TVTs, TMOs = [], []
for ids in sys_device_ids:
relative_ids = []
for id in ids:
if id != -1:
id = find_index(unique_sys_device_ids, id)
relative_ids.append(id)
relative_device_ids.append(relative_ids)
# Models and user defined Variables/Tensors would be transferred to the
# first device.
TVTs.append(TransferVarTensor(relative_ids[0]))
TMOs.append(TransferModulesOptims(relative_ids[0]))
return TVTs, TMOs, relative_device_ids
def load_ckpt(modules_optims, ckpt_file, load_to_cpu=True, verbose=True):
"""Load state_dict's of modules/optimizers from file.
Args:
modules_optims: A list, which members are either torch.nn.optimizer
or torch.nn.Module.
ckpt_file: The file path.
load_to_cpu: Boolean. Whether to transform tensors in modules/optimizers
to cpu type.
"""
map_location = (lambda storage, loc: storage) if load_to_cpu else None
ckpt = torch.load(ckpt_file, map_location=map_location)
for m, sd in zip(modules_optims, ckpt['state_dicts']):
if 'fc.weight' in sd:
del sd['fc.weight']
if 'fc.bias' in sd:
del sd['fc.bias']
load_state_dict(m, sd)
if verbose:
print('Resume from ckpt {}, \nepoch {}, \nscores {}'.format(
ckpt_file, ckpt['ep'], ckpt['scores']))
return ckpt['ep'], ckpt['scores']
def save_ckpt(modules_optims, ep, scores, ckpt_file):
"""Save state_dict's of modules/optimizers to file.
Args:
modules_optims: A list, which members are either torch.nn.optimizer
or torch.nn.Module.
ep: the current epoch number
scores: the performance of current model
ckpt_file: The file path.
Note:
torch.save() reserves device type and id of tensors to save, so when
loading ckpt, you have to inform torch.load() to load these tensors to
cpu or your desired gpu, if you change devices.
"""
state_dicts = [m.state_dict() for m in modules_optims]
ckpt = dict(state_dicts=state_dicts,
ep=ep,
scores=scores)
may_make_dir(osp.dirname(osp.abspath(ckpt_file)))
torch.save(ckpt, ckpt_file)
def load_state_dict(model, src_state_dict):
"""Copy parameters and buffers from `src_state_dict` into `model` and its
descendants. The `src_state_dict.keys()` NEED NOT exactly match
`model.state_dict().keys()`. For dict key mismatch, just
skip it; for copying error, just output warnings and proceed.
Arguments:
model: A torch.nn.Module object.
src_state_dict (dict): A dict containing parameters and persistent buffers.
Note:
This is modified from torch.nn.modules.module.load_state_dict(), to make
the warnings and errors more detailed.
"""
from torch.nn import Parameter
dest_state_dict = model.state_dict()
for name, param in src_state_dict.items():
### CHANGED HERE FOR FINE TUNING
if name not in dest_state_dict:
continue
if isinstance(param, Parameter):
# backwards compatibility for serialized parameters
param = param.data
try:
dest_state_dict[name].copy_(param)
except Exception as e:
print("Warning: Error occurs when copying '{}': {}"
.format(name, str(e)))
# src_missing = set(dest_state_dict.keys()) - set(src_state_dict.keys())
# if len(src_missing) > 0:
# print("Keys not found in source state_dict: ")
# for n in src_missing:
# print('\t', n)
# dest_missing = set(src_state_dict.keys()) - set(dest_state_dict.keys())
# if len(dest_missing) > 0:
# print("Keys not found in destination state_dict: ")
# for n in dest_missing:
# print('\t', n)
def is_iterable(obj):
return hasattr(obj, '__len__')
def may_set_mode(maybe_modules, mode):
"""maybe_modules: an object or a list of objects."""
assert mode in ['train', 'eval']
if not is_iterable(maybe_modules):
maybe_modules = [maybe_modules]
for m in maybe_modules:
if isinstance(m, torch.nn.Module):
if mode == 'train':
m.train()
else:
m.eval()
def may_make_dir(path):
"""
Args:
path: a dir, or result of `osp.dirname(osp.abspath(file_path))`
Note:
`osp.exists('')` returns `False`, while `osp.exists('.')` returns `True`!
"""
# This clause has mistakes:
# if path is None or '':
if path in [None, '']:
return
if not osp.exists(path):
os.makedirs(path)
class AverageMeter(object):
"""Modified from Tong Xiao's open-reid.
Computes and stores the average and current value"""
def __init__(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = float(self.sum) / (self.count + 1e-20)
class RunningAverageMeter(object):
"""Computes and stores the running average and current value"""
def __init__(self, hist=0.99):
self.val = None
self.avg = None
self.hist = hist
def reset(self):
self.val = None
self.avg = None
def update(self, val):
if self.avg is None:
self.avg = val
else:
self.avg = self.avg * self.hist + val * (1 - self.hist)
self.val = val
class RecentAverageMeter(object):
"""Stores and computes the average of recent values."""
def __init__(self, hist_size=100):
self.hist_size = hist_size
self.fifo = []
self.val = 0
def reset(self):
self.fifo = []
self.val = 0
def update(self, val):
self.val = val
self.fifo.append(val)
if len(self.fifo) > self.hist_size:
del self.fifo[0]
@property
def avg(self):
assert len(self.fifo) > 0
return float(sum(self.fifo)) / len(self.fifo)
def get_model_wrapper(model, multi_gpu):
from torch.nn.parallel import DataParallel
if multi_gpu:
return DataParallel(model)
else:
return model
class ReDirectSTD(object):
"""Modified from Tong Xiao's `Logger` in open-reid.
This class overwrites sys.stdout or sys.stderr, so that console logs can
also be written to file.
Args:
fpath: file path
console: one of ['stdout', 'stderr']
immediately_visible: If `False`, the file is opened only once and closed
after exiting. In this case, the message written to file may not be
immediately visible (Because the file handle is occupied by the
program?). If `True`, each writing operation of the console will
open, write to, and close the file. If your program has tons of writing
operations, the cost of opening and closing file may be obvious. (?)
Usage example:
`ReDirectSTD('stdout.txt', 'stdout', False)`
`ReDirectSTD('stderr.txt', 'stderr', False)`
NOTE: File will be deleted if already existing. Log dir and file is created
lazily -- if no message is written, the dir and file will not be created.
"""
def __init__(self, fpath=None, console='stdout', immediately_visible=False):
import sys
import os
import os.path as osp
assert console in ['stdout', 'stderr']
self.console = sys.stdout if console == 'stdout' else sys.stderr
self.file = fpath
self.f = None
self.immediately_visible = immediately_visible
if fpath is not None:
# Remove existing log file.
if osp.exists(fpath):
os.remove(fpath)
# Overwrite
if console == 'stdout':
sys.stdout = self
else:
sys.stderr = self
def __del__(self):
self.close()
def __enter__(self):
pass
def __exit__(self, *args):
self.close()
def write(self, msg):
self.console.write(msg)
if self.file is not None:
may_make_dir(os.path.dirname(osp.abspath(self.file)))
if self.immediately_visible:
with open(self.file, 'a') as f:
f.write(msg)
else:
if self.f is None:
self.f = open(self.file, 'w')
self.f.write(msg)
def flush(self):
self.console.flush()
if self.f is not None:
self.f.flush()
import os
os.fsync(self.f.fileno())
def close(self):
self.console.close()
if self.f is not None:
self.f.close()
def set_seed(seed):
import random
random.seed(seed)
print('setting random-seed to {}'.format(seed))
import numpy as np
np.random.seed(seed)
print('setting np-random-seed to {}'.format(seed))
import torch
torch.backends.cudnn.enabled = False
print('cudnn.enabled set to {}'.format(torch.backends.cudnn.enabled))
# set seed for CPU
torch.manual_seed(seed)
print('setting torch-seed to {}'.format(seed))
def print_array(array, fmt='{:.2f}', end=' '):
"""Print a 1-D tuple, list, or numpy array containing digits."""
s = ''
for x in array:
s += fmt.format(float(x)) + end
s += '\n'
print(s)
return s
# Great idea from https://github.com/amdegroot/ssd.pytorch
def str2bool(v):
return v.lower() in ("yes", "true", "t", "1")
def tight_float_str(x, fmt='{:.4f}'):
return fmt.format(x).rstrip('0').rstrip('.')
def find_index(seq, item):
for i, x in enumerate(seq):
if item == x:
return i
return -1
def adjust_lr_exp(optimizer, base_lr, ep, total_ep, start_decay_at_ep):
"""Decay exponentially in the later phase of training. All parameters in the
optimizer share the same learning rate.
Args:
optimizer: a pytorch `Optimizer` object
base_lr: starting learning rate
ep: current epoch, ep >= 1
total_ep: total number of epochs to train
start_decay_at_ep: start decaying at the BEGINNING of this epoch
Example:
base_lr = 2e-4
total_ep = 300
start_decay_at_ep = 201
It means the learning rate starts at 2e-4 and begins decaying after 200
epochs. And training stops after 300 epochs.
NOTE:
It is meant to be called at the BEGINNING of an epoch.
"""
assert ep >= 1, "Current epoch number should be >= 1"
if ep < start_decay_at_ep:
return
for g in optimizer.param_groups:
g['lr'] = (base_lr * (0.001 ** (float(ep + 1 - start_decay_at_ep)
/ (total_ep + 1 - start_decay_at_ep))))
print('=====> lr adjusted to {:.10f}'.format(g['lr']).rstrip('0'))
def adjust_lr_staircase(optimizer, base_lr, ep, decay_at_epochs, factor):
"""Multiplied by a factor at the BEGINNING of specified epochs. All
parameters in the optimizer share the same learning rate.
Args:
optimizer: a pytorch `Optimizer` object
base_lr: starting learning rate
ep: current epoch, ep >= 1
decay_at_epochs: a list or tuple; learning rate is multiplied by a factor
at the BEGINNING of these epochs
factor: a number in range (0, 1)
Example:
base_lr = 1e-3
decay_at_epochs = [51, 101]
factor = 0.1
It means the learning rate starts at 1e-3 and is multiplied by 0.1 at the
BEGINNING of the 51'st epoch, and then further multiplied by 0.1 at the
BEGINNING of the 101'st epoch, then stays unchanged till the end of
training.
NOTE:
It is meant to be called at the BEGINNING of an epoch.
"""
assert ep >= 1, "Current epoch number should be >= 1"
if ep not in decay_at_epochs:
return
ind = find_index(decay_at_epochs, ep)
for g in optimizer.param_groups:
g['lr'] = base_lr * factor ** (ind + 1)
print('=====> lr adjusted to {:.10f}'.format(g['lr']).rstrip('0'))
@contextmanager
def measure_time(enter_msg):
st = time.time()
print(enter_msg)
yield
print('Done, {:.2f}s'.format(time.time() - st))
# @profile
def generate_features(appearance_model, patches, opt, object_ids = None):
Tensor = torch.cuda.FloatTensor if opt.using_cuda else torch.FloatTensor
features = []
for i, patch in enumerate(patches):
if patch is None or patch.nelement()==0:
features.append(None)
continue
patch = patch.unsqueeze(0)
if opt.perfect:
feature = torch.zeros(1024)
feature[object_ids[i]] = 1
feature = feature.type(Tensor)
else:
if opt.appearance_model == 'aligned_reid':
with torch.no_grad():
feature ,_ = appearance_model(patch.cuda())
feature = feature.squeeze(0).type(Tensor)
elif opt.appearance_model == 'deepsort':
patch = patch.permute(0,2,3,1).cpu().numpy()
feature = appearance_model(patch)
feature = feature[0]
elif opt.appearance_model == 'resnet_reid':
patch = patch.permute(0,2,3,1)
feature = appearance_model.inference([patch.squeeze(0)])
feature = feature[0][0].type(Tensor)
features.append(feature)
return features
# @profile
def generate_features_batched(appearance_model, patches, opt, object_ids = None):
if opt.perfect or opt.appearance_model == 'deepsort': # Do old/slow way if perfect features or deepsort features
return generate_features(appearance_model, patches, opt, object_ids)
if opt.appearance_model == 'resnet_reid':
Tensor = torch.cuda.FloatTensor if opt.using_cuda else torch.FloatTensor
features = []
resnet_patches = []
for i, patch in enumerate(patches):
if patch is None or patch.nelement()==0:
features.append(None)
else:
features.append(1)
resnet_patches.append(patch.permute(1,2,0))
resnet_features = appearance_model.inference(resnet_patches)
ctr = 0
for i in range(len(features)):
if features[i] is not None:
features[i] = resnet_features[ctr].type(Tensor)
ctr += 1
return features
elif opt.appearance_model == 'aligned_reid':
return generate_features(appearance_model, patches, opt, object_ids) #TODO: Fix batched appearance features. This currently gives bad features
Tensor = torch.cuda.FloatTensor if opt.using_cuda else torch.FloatTensor
maxx = -1
maxy = -1
idxs = []
features = []
for i, patch in enumerate(patches):
if patch is None or patch.nelement()==0:
continue
maxx = max(maxx, patch.size()[1])
maxy = max(maxy, patch.size()[2])
idxs.append(i)
if(maxx==-1 and maxy==-1):
return features
batch = torch.zeros(len(idxs),3,maxx,maxy).cuda()
for i, idx in enumerate(idxs):
patch = patches[idx]
patchx = patch.size()[1]
patchy = patch.size()[2]
batch[i,:,:patchx,:patchy] = patch
with torch.no_grad():
features_torch, _ = appearance_model(batch)
features_torch = features_torch.type(Tensor)
i = 0
ctr = 0
for idx in idxs:
while(i < idx):
features.append(None)
i+=1
features.append(features_torch[ctr,:])
i+=1
ctr+=1
while(i<len(patches)):
features.append(None)
i+=1
return features
else:
print("Critical Error! Attempted to batch appearance features but no model was selected")
def get_image_patches(input_img, detections):
#Generates patches and also converts detections
patches = []
for detection in detections:
x1, y1, x2, y2, _, _, _ = detection
# Rescale coordinates to original dimensions
x1 = x1.item()
x2 = x2.item()
y1 = y1.item()
y2 = y2.item()
box_h = round(y2-y1)
box_w = round(x2-x1)
x1=round(x1)
y1=round(y1)
patch = input_img[:, y1:y1+box_h, x1:x1+box_w]
patches.append(patch)
return patches
def create_appearance_model(model_type, alignreid_checkpoint, resnet_reid_checkpoint=None, cuda=True):
if model_type == 'aligned_reid':
appearance_model = aligned_reid_model()
map_location = (lambda storage, loc: storage)
sd = torch.load(alignreid_checkpoint, map_location=map_location)
load_state_dict(appearance_model, sd['state_dicts'][0])
if cuda:
appearance_model.cuda()
appearance_model.eval()
elif model_type == 'deepsort':
appearance_model = deep_sort_model()
elif model_type == 'resnet_reid':
appearance_model = ResNet_Loader(resnet_reid_checkpoint)
return appearance_model
================================================
FILE: paper_experiments/utils/assign_ids_detections.py
================================================
import numpy as np
import os
import pdb
from tqdm import tqdm
from deep_sort_utils import non_max_suppression as deepsort_nms
from visualise import draw_track
import matplotlib.pyplot as plt
from PIL import Image
from evaluate_detections import iou
def assign_detection_id(detection_path, gt_path, conf_threshold = 0, iou_threshold = 0.5):
#expecting detections and gt in file with format as in read_detections.py
# applies confidence thresholding
try:
detections = np.loadtxt(detection_path, delimiter=',')
gt = np.loadtxt(gt_path, delimiter=',')
except:
return
gt_frames = gt[:, 0]
det_confidence = detections[:, 6]
###CONFIDENCE THRESHOLD
detections = detections[det_confidence > conf_threshold]
########
det_frames = detections[:, 0]
det_confidence = detections[:, 6]
gt_boxes = np.asarray(list(zip(gt[:, 2], gt[:, 3], gt[:, 4], gt[:, 5])))
det_boxes = np.asarray(list(zip(detections[:, 2], detections[:, 3], detections[:, 4], detections[:, 5])))
out_matrix = []
assigned_ids = []
for frame in np.unique(det_frames):
frame_mask_det = det_frames == frame
frame_mask_gt = gt_frames == frame
gt_ids = gt[frame_mask_gt, 1]
frame_gt_boxes = gt_boxes[frame_mask_gt]
frame_det_boxes = det_boxes[frame_mask_det]
for i, det_box in enumerate(frame_det_boxes):
iou_list = np.asarray([iou(gt_box, det_box) for gt_box in frame_gt_boxes])
iou_sorted = np.argsort(iou_list)
positive_idx = np.where(iou_list >= iou_threshold)[0]
if len(positive_idx)==0:
assigned_ids.append(-1)
else:
assigned_ids.append(gt_ids[iou_sorted[-1]])
assigned_ids = np.expand_dims(np.asarray(assigned_ids), 1)
try:
out_matrix = np.hstack([np.expand_dims(detections[:,0], 1), assigned_ids, detections[:,2:]])
except:
pdb.set_trace()
np.savetxt(detection_path, out_matrix, delimiter=',', fmt = '%.2f')
return
if __name__=='__main__':
ap = []
KITTI_root = 'data/KITTI/sequences'
for sequence in tqdm(range(21)):
assign_detection_id(os.path.join(KITTI_root, '%.4d'%sequence, 'det','rrc_subcnn_car_det.txt'),
os.path.join(KITTI_root, '%.4d'%sequence, 'gt', 'gt_car.txt'))
================================================
FILE: paper_experiments/utils/calibration.py
================================================
import numpy as np
import cv2
import os
import yaml
import torch
import pdb
class Calibration(object):
''' Calibration matrices and utils
3d XYZ in <label>.txt are in rect camera coord.
2d box xy are in image2 coord
Points in <lidar>.bin are in Velodyne coord.
y_image2 = P^2_rect * x_rect
y_image2 = P^2_rect * R0_rect * Tr_velo_to_cam * x_velo
x_ref = Tr_velo_to_cam * x_velo
x_rect = R0_rect * x_ref
P^2_rect = [f^2_u, 0, c^2_u, -f^2_u b^2_x;
0, f^2_v, c^2_v, -f^2_v b^2_y;
0, 0, 1, 0]
= K * [1|t]
image2 coord:
----> x-axis (u)
|
|
v y-axis (v)
velodyne coord:
front x, left y, up z
rect/ref camera coord:
right x, down y, front z
Ref (KITTI paper): http://www.cvlibs.net/publications/Geiger2013IJRR.pdf
TODO(rqi): do matrix multiplication only once for each projection.
'''
def __init__(self, calib_filepath):
calibs = self.read_calib_file(calib_filepath)
# Projection matrix from rect camera coord to image2 coord
self.P = calibs['P2']
self.P = np.reshape(self.P, [3,4])
self.P_torch = torch.from_numpy(self.P).float().cuda()
# Rigid transform from Velodyne coord to reference camera coord
try:
self.V2C = calibs['Tr_velo_to_cam']
except:
self.V2C = calibs['Tr_velo_cam']
self.V2C = np.reshape(self.V2C, [3,4])
self.C2V = inverse_rigid_trans(self.V2C)
# Rotation from reference camera coord to rect camera coord
try:
self.R0 = calibs['R0_rect']
except:
self.R0 = calibs['R_rect']
self.R0 = np.reshape(self.R0,[3,3])
self.R0_torch = torch.from_numpy(self.R0).float().cuda()
RA = np.zeros((4,4))
RA[:3,:3] = self.R0
RA[3,3] = 1
self.D = np.matmul(self.P,RA).T
self.D_torch = torch.from_numpy(self.D).float().cuda()
# Camera intrinsics and extrinsics
self.c_u = self.P[0,2]
self.c_v = self.P[1,2]
self.f_u = self.P[0,0]
self.f_v = self.P[1,1]
self.b_x = self.P[0,3]/(-self.f_u) # relative
self.b_y = self.P[1,3]/(-self.f_v)
def read_calib_file(self, filepath):
''' Read in a calibration file and parse into a dictionary.
Ref: https://github.com/utiasSTARS/pykitti/blob/master/pykitti/utils.py
'''
data = {}
with open(filepath, 'r') as f:
for line in f.readlines():
line = line.rstrip()
if len(line)==0: continue
key, value = line.split(' ', 1)
if key.endswith(':'):
key = key[:-1]
# The only non-float values in these files are dates, which
# we don't care about anyway
try:
data[key] = np.array([float(x) for x in value.split()])
except ValueError:
pass
return data
def read_calib_from_video(self, calib_root_dir):
''' Read calibration for camera 2 from video calib files.
there are calib_cam_to_cam and calib_velo_to_cam under the calib_root_dir
'''
data = {}
cam2cam = self.read_calib_file(os.path.join(calib_root_dir, 'calib_cam_to_cam.txt'))
velo2cam = self.read_calib_file(os.path.join(calib_root_dir, 'calib_velo_to_cam.txt'))
Tr_velo_to_cam = np.zeros((3,4))
Tr_velo_to_cam[0:3,0:3] = np.reshape(velo2cam['R'], [3,3])
Tr_velo_to_cam[:,3] = velo2cam['T']
data['Tr_velo_to_cam'] = np.reshape(Tr_velo_to_cam, [12])
data['R0_rect'] = cam2cam['R_rect_00']
data['P2'] = cam2cam['P_rect_02']
return data
def cart2hom(self, pts_3d):
''' Input: nx3 points in Cartesian
Oupput: nx4 points in Homogeneous by appending 1
'''
n = pts_3d.shape[0]
pts_3d_hom = np.hstack((pts_3d, np.ones((n,1))))
return pts_3d_hom
def cart2hom_torch(self, pts_3d):
n = pts_3d.size()[0]
pts_3d_hom = torch.cat((pts_3d, torch.ones(n,1).to("cuda:0")), 1)
return pts_3d_hom
# ===========================
# ------- 3d to 3d ----------
# ===========================
def project_velo_to_ref(self, pts_3d_velo):
pts_3d_velo = self.cart2hom(pts_3d_velo) # nx4
return np.dot(pts_3d_velo, np.transpose(self.V2C))
def project_ref_to_velo(self, pts_3d_ref):
pts_3d_ref = self.cart2hom(pts_3d_ref) # nx4
return np.dot(pts_3d_ref, np.transpose(self.C2V))
def project_rect_to_ref(self, pts_3d_rect):
''' Input and Output are nx3 points '''
return np.transpose(np.dot(np.linalg.inv(self.R0), np.transpose(pts_3d_rect)))
def project_ref_to_rect(self, pts_3d_ref):
''' Input and Output are nx3 points '''
return np.transpose(np.dot(self.R0, np.transpose(pts_3d_ref)))
def project_ref_to_rect_torch(self, pts_3d_ref):
''' Input and Output are nx3 points '''
return torch.transpose(torch.matmul(self.R0_torch, torch.transpose(pts_3d_ref,0,1)),0,1)
def project_rect_to_velo(self, pts_3d_rect):
''' Input: nx3 points in rect camera coord.
Output: nx3 points in velodyne coord.
'''
pts_3d_ref = self.project_rect_to_ref(pts_3d_rect)
return self.project_ref_to_velo(pts_3d_ref)
def project_velo_to_rect(self, pts_3d_velo):
pts_3d_ref = self.project_velo_to_ref(pts_3d_velo)
return self.project_ref_to_rect(pts_3d_ref)
# ===========================
# ------- 3d to 2d ----------
# ===========================
def project_rect_to_image(self, pts_3d_rect):
''' Input: nx3 points in rect camera coord.
Output: nx2 points in image2 coord.
'''
pts_3d_rect = self.cart2hom(pts_3d_rect)
pts_2d = np.dot(pts_3d_rect, np.transpose(self.P)) # nx3
pts_2d[:,0] /= pts_2d[:,2]
pts_2d[:,1] /= pts_2d[:,2]
return pts_2d[:,0:2]
def project_rect_to_image_torch(self, pts_3d_rect):
''' Input: nx3 points in rect camera coord.
Output: nx2 points in image2 coord.
'''
pts_3d_rect = self.cart2hom_torch(pts_3d_rect)
pts_2d = torch.matmul(pts_3d_rect, torch.transpose(self.P_torch,0,1)) # nx3
pts_2d[:,0] /= pts_2d[:,2]
pts_2d[:,1] /= pts_2d[:,2]
return pts_2d[:,0:2]
def project_ref_to_image_torch(self, pts_3d_ref):
''' Input: nx3 points in ref camera coord.
Output: nx2 points in image2 coord.
'''
pts_3d_ref = self.cart2hom_torch(pts_3d_ref)
pts_2d = torch.matmul(pts_3d_ref, self.D_torch) # nx3
pts_2d[:,0] /= pts_2d[:,2]
pts_2d[:,1] /= pts_2d[:,2]
return pts_2d[:,0:2]
def project_velo_to_image(self, pts_3d_velo):
''' Input: nx3 points in velodyne coord.
Output: nx2 points in image2 coord.
'''
pts_3d_rect = self.project_velo_to_rect(pts_3d_velo)
return self.project_rect_to_image(pts_3d_rect)
# ===========================
# ------- 2d to 3d ----------
# ===========================
def project_image_to_rect(self, uv_depth):
''' Input: nx3 first two channels are uv, 3rd channel
is depth in rect camera coord.
Output: nx3 points in rect camera coord.
'''
n = uv_depth.shape[0]
x = ((uv_depth[:,0]-self.c_u)*uv_depth[:,2])/self.f_u + self.b_x
y = ((uv_depth[:,1]-self.c_v)*uv_depth[:,2])/self.f_v + self.b_y
pts_3d_rect = np.zeros((n,3))
pts_3d_rect[:,0] = x
pts_3d_rect[:,1] = y
pts_3d_rect[:,2] = uv_depth[:,2]
return pts_3d_rect
def project_image_to_velo(self, uv_depth):
pts_3d_rect = self.project_image_to_rect(uv_depth)
return self.project_rect_to_velo(pts_3d_rect)
def rotx(t):
''' 3D Rotation about the x-axis. '''
c = np.cos(t)
s = np.sin(t)
return np.array([[1, 0, 0],
[0, c, -s],
[0, s, c]])
def roty(t):
''' Rotation about the y-axis. '''
c = np.cos(t)
s = np.sin(t)
return np.array([[c, 0, s],
[0, 1, 0],
[-s, 0, c]])
def rotz(t):
''' Rotation about the z-axis. '''
c = np.cos(t)
s = np.sin(t)
return np.array([[c, -s, 0],
[s, c, 0],
[0, 0, 1]])
def transform_from_rot_trans(R, t):
''' Transforation matrix from rotation matrix and translation vector. '''
R = R.reshape(3, 3)
t = t.reshape(3, 1)
return np.vstack((np.hstack([R, t]), [0, 0, 0, 1]))
def inverse_rigid_trans(Tr):
''' Inverse a rigid body transform matrix (3x4 as [R|t])
[R'|-R't; 0|1]
'''
inv_Tr = np.zeros_like(Tr) # 3x4
inv_Tr[0:3,0:3] = np.transpose(Tr[0:3,0:3])
inv_Tr[0:3,3] = np.dot(-np.transpose(Tr[0:3,0:3]), Tr[0:3,3])
return inv_Tr
def read_label(label_filename):
lines = [line.rstrip() for line in open(label_filename)]
objects = [Object3d(line) for line in lines]
return objects
def load_image(img_filename):
return cv2.imread(img_filename)
def load_velo_scan(velo_filename):
scan = np.fromfile(velo_filename, dtype=np.float32)
scan = scan.reshape((-1, 4))
return scan
def project_to_image(pts_3d, P):
''' Project 3d points to image plane.
Usage: pts_2d = projectToImage(pts_3d, P)
input: pts_3d: nx3 matrix
P: 3x4 projection matrix
output: pts_2d: nx2 matrix
P(3x4) dot pts_3d_extended(4xn) = projected_pts_2d(3xn)
=> normalize projected_pts_2d(2xn)
<=> pts_3d_extended(nx4) dot P'(4x3) = projected_pts_2d(nx3)
=> normalize projected_pts_2d(nx2)
'''
n = pts_3d.shape[0]
pts_3d_extend = np.hstack((pts_3d, np.ones((n,1))))
print(('pts_3d_extend shape: ', pts_3d_extend.shape))
pts_2d = np.dot(pts_3d_extend, np.transpose(P)) # nx3
pts_2d[:,0] /= pts_2d[:,2]
pts_2d[:,1] /= pts_2d[:,2]
return pts_2d[:,0:2]
def compute_box_3d(obj, P):
''' Takes an object and a projection matrix (P) and projects the 3d
bounding box into the image plane.
Returns:
corners_2d: (8,2) array in left image coord.
corners_3d: (8,3) array in in rect camera coord.
'''
# compute rotational matrix around yaw axis
R = roty(obj.ry)
# 3d bounding box dimensions
l = obj.l;
w = obj.w;
h = obj.h;
# 3d bounding box corners
x_corners = [l/2,l/2,-l/2,-l/2,l/2,l/2,-l/2,-l/2];
y_corners = [0,0,0,0,-h,-h,-h,-h];
z_corners = [w/2,-w/2,-w/2,w/2,w/2,-w/2,-w/2,w/2];
# rotate and translate 3d bounding box
corners_3d = np.dot(R, np.vstack([x_corners,y_corners,z_corners]))
#print corners_3d.shape
corners_3d[0,:] = corners_3d[0,:] + obj.t[0];
corners_3d[1,:] = corners_3d[1,:] + obj.t[1];
corners_3d[2,:] = corners_3d[2,:] + obj.t[2];
#print 'cornsers_3d: ', corners_3d
# only draw 3d bounding box for objs in front of the camera
if np.any(corners_3d[2,:]<0.1):
corners_2d = None
return corners_2d, np.transpose(corners_3d)
# project the 3d bounding box into the image plane
corners_2d = project_to_image(np.transpose(corners_3d), P);
#print 'corners_2d: ', corners_2d
return corners_2d, np.transpose(corners_3d)
def compute_orientation_3d(obj, P):
''' Takes an object and a projection matrix (P) and projects the 3d
object orientation vector into the image plane.
Returns:
orientation_2d: (2,2) array in left image coord.
orientation_3d: (2,3) array in in rect camera coord.
'''
# compute rotational matrix around yaw axis
R = roty(obj.ry)
# orientation in object coordinate system
orientation_3d = np.array([[0.0, obj.l],[0,0],[0,0]])
# rotate and translate in camera coordinate system, project in image
orientation_3d = np.dot(R, orientation_3d)
orientation_3d[0,:] = orientation_3d[0,:] + obj.t[0]
orientation_3d[1,:] = orientation_3d[1,:] + obj.t[1]
orientation_3d[2,:] = orientation_3d[2,:] + obj.t[2]
# vector behind image plane?
if np.any(orientation_3d[2,:]<0.1):
orientation_2d = None
return orientation_2d, np.transpose(orientation_3d)
# project orientation into the image plane
orientation_2d = project_to_image(np.transpose(orientation_3d), P);
return orientation_2d, np.transpose(orientation_3d)
def draw_projected_box3d(image, qs, color=(255,255,255), thickness=2):
''' Draw 3d bounding box in image
qs: (8,3) array of vertices for the 3d box in following order:
1 -------- 0
/| /|
2 -------- 3 .
| | | |
. 5 -------- 4
|/ |/
6 -------- 7
'''
qs = qs.astype(np.int32)
for k in range(0,4):
# Ref: http://docs.enthought.com/mayavi/mayavi/auto/mlab_helper_functions.html
i,j=k,(k+1)%4
# use LINE_AA for opencv3
cv2.line(image, (qs[i,0],qs[i,1]), (qs[j,0],qs[j,1]), color, thickness, cv2.CV_AA)
i,j=k+4,(k+1)%4 + 4
cv2.line(image, (qs[i,0],qs[i,1]), (qs[j,0],qs[j,1]), color, thickness, cv2.CV_AA)
i,j=k,k+4
cv2.line(image, (qs[i,0],qs[i,1]), (qs[j,0],qs[j,1]), color, thickness, cv2.CV_AA)
return image
class OmniCalibration(Calibration):
def __init__(self, calib_folder):
global_config = os.path.join(calib_folder, 'defaults.yaml')
camera_config = os.path.join(calib_folder, 'cameras.yaml')
with open(global_config) as f:
self.global_config_dict = yaml.safe_load(f)
with open(camera_config) as f:
self.camera_config_dict = yaml.safe_load(f)
self.median_focal_length_y = self.calculate_median_param_value(param = 'f_y')
self.median_optical_center_y = self.calculate_median_param_value(param = 't_y')
# image shape is (color channels, height, width)
self.img_shape = 3, self.global_config_dict['image']['height'], self.global_config_dict['image']['width']
def project_ref_to_image_torch(self, pointcloud):
theta = (torch.atan2(pointcloud[:, 0], pointcloud[:, 2]) + np.pi) %(2*np.pi)
horizontal_fraction = theta/ (2*np.pi)
x = (horizontal_fraction * self.img_shape[2]) % self.img_shape[2]
y = -self.median_focal_length_y*(pointcloud[:, 1]*torch.cos(theta)/pointcloud[:, 2]) + self.median_optical_center_y
pts_2d = torch.stack([x, y], dim=1)
return pts_2d
def project_image_to_rect(self, uvdepth):
theta = (uvdepth[:, 0]/self.img_shape[2])*2*np.pi - np.pi
z = uvdepth[:, 2]*np.cos(theta)
x = uvdepth[:, 2]*np.sin(theta)
y = z*-1*(uvdepth[:, 1] - self.median_optical_center_y)/(self.median_focal_length_y * np.cos(theta))
return np.stack([x,y,z], axis=1)
def project_velo_to_ref(self, pointcloud):
pointcloud = pointcloud[:, [1, 2, 0]]
pointcloud[:, 0] *= -1
pointcloud[:, 1] *= -1
return pointcloud
def move_lidar_to_camera_frame(self, pointcloud, upper = True):
# assumed only rotation about z axis
if upper:
pointcloud -= self.global_config_dict['lidar_upper_to_rgb']['translation']
theta = self.global_config_dict['lidar_upper_to_rgb']['rotation'][-1]
else:
pointcloud -= self.global_config_dict['lidar_lower_to_rgb']['translation']
theta = self.global_config_dict['lidar_lower_to_rgb']['rotation'][-1]
rotation_matrix = torch.Tensor([[np.cos(theta), np.sin(theta)], [-np.sin(theta), np.cos(theta)]]).type(pointcloud.type())
pointcloud[:, :2] = torch.matmul(rotation_matrix.unsqueeze(0), pointcloud[:, :2].transpose(0,1)).transpose(0,1)
return pointcloud
def calculate_median_param_value(self, param):
if param=='f_y':
idx = 4
elif param == 'f_x':
idx = 0
elif param == 't_y':
idx = 5
elif param == 't_x':
idx = 2
elif param == 's':
idx = 1
else:
raise 'Wrong parameter!'
omni_camera = ['sensor_0', 'sensor_2', 'sensor_4', 'sensor_6', 'sensor_8']
parameter_list = []
for sensor, camera_params in self.camera_config_dict['cameras'].items():
if sensor not in omni_camera:
continue
K_matrix = camera_params['K'].split(' ')
parameter_list.append(float(K_matrix[idx]))
return np.median(parameter_list)
================================================
FILE: paper_experiments/utils/combine_and_process_detections.py
================================================
import os
from os import listdir
from os.path import isfile, join
#root = "/cvgl2/u/mihirp/depth_tracking/data/JRDB/sequences/"
#root = "/cvgl2/u/mihirp/depth_tracking/data/JRDB/test_sequences/"
#root = "/cvgl2/u/mihirp/depth_tracking/data/KITTI/sequences/"
root = "/cvgl2/u/mihirp/depth_tracking/data/KITTI/test_sequences/"
file_name = "new_subcnn_faster_rcnn"
# file_name = "detectron2_x101"
def threshold(filename, thresh, min, max):
detections = []
with open(filename, 'r') as readfile:
dets = readfile.read().split('\n')
dets = dets[:len(dets)-1] #filter out last line which is just \n
for det in dets:
parsedet = det.split(' ')
score = float(parsedet[len(parsedet)-1])
parsedet[len(parsedet)-1] = str((float(parsedet[len(parsedet)-1]) - thresh) / (max - thresh))
if(score > thresh):
detections.append(parsedet)
return detections
for seq in sorted(os.listdir(root)): #21 for normal, 29 for testing
path = os.path.join(root,seq,'det')
with open(os.path.join(path,file_name+'_raw.txt'), 'w') as f:
pred_dets = []
#pred_dets.append(threshold(os.path.join(path,'rrc.txt'), .05, 0, 1))
pred_dets.append(threshold(os.path.join(path,'subcnn.txt'), .8, 0, 1))
#pred_dets.append(threshold(os.path.join(path,'faster_rcnn.txt'), .99, 0, 1))
#pred_dets.append(threshold(os.path.join(path,'detectron2_x101.txt'), .9, 0, 1))
#pred_dets.append(threshold(path+'regionlets.txt', 5, -5, 25))
if len(pred_dets[0]) == 0:
continue
max_frames = int((pred_dets[0])[len(pred_dets[0])-1][0])
det_ctrs = [0,0,0,0]
for frame in range(max_frames+1):
frame_num = 0
for j in range(1): #TODO: Update to number of detectors used
while det_ctrs[j] < len(pred_dets[j]) and int( (pred_dets[j])[det_ctrs[j]][0]) == frame:
(pred_dets[j])[det_ctrs[j]][1] = str(frame_num)
frame_num+=1
f.write( " ".join( (pred_dets[j])[det_ctrs[j]] )+'\n')
det_ctrs[j]+=1
# Counts max/min of scores
for ctr, pred_det in enumerate(pred_dets):
minval = 1000
maxval = 0
for detection in pred_det:
score = detection[len(detection)-1]
if float(score)>maxval:
maxval = float(score)
if float(score)<minval:
minval = float(score)
# print("Detector: "+str(ctr)+" Max: "+str(maxval))
# print("Detector: "+str(ctr)+" Min: "+str(minval))
with open(os.path.join(path,file_name+'_raw.txt'), 'r') as f:
lines = f.readlines()
with open(os.path.join(path, file_name+'_car.txt'), 'w') as fcar:
with open(os.path.join(path, file_name+'_ped.txt'), 'w') as fped:
for line in lines:
if len(line) < 5:
continue
vals = line.split(' ')
min_x = float(vals[6])
min_y = float(vals[7])
max_x = float(vals[8])
max_y = float(vals[9])
score = vals[-1]
out_line = vals[0]+',0,'+str(min_x)+','+str(min_y)+','+str(max_x-min_x)+','+str(max_y-min_y)+','+str(score)
if vals[2] == 'Car':
fcar.write(out_line)
elif vals[2] == 'Pedestrian':
fped.write(out_line)
================================================
FILE: paper_experiments/utils/dataset.py
================================================
import glob
import os
import pdb
import random
import sys
from itertools import compress
import numpy as np
import torch
import torchvision.transforms as transforms
from PIL import Image
from skimage.transform import resize
from torch.utils.data import Dataset
from tqdm import tqdm
from models.pointnet_model import PointNet
from .calibration import Calibration, OmniCalibration
from .read_detections import (read_ground_truth_2d_detections,
read_ground_truth_3d_detections)
class SequenceDataset(Dataset):
def __init__(self, folder_path, point_cloud=False, cuda=False, omni=False):
self.files = sorted(glob.glob('%s/imgs/*.*' % folder_path), key = lambda x: int(os.path.splitext(os.path.basename(x))[0]))
self.files = [file for file in self.files if is_image_file(file)]
self.point_cloud = point_cloud
self.seq_name = os.path.split(folder_path)[-1]
self.omni = omni
if point_cloud:
if self.omni:
calib_folder = os.path.join(folder_path, 'calib')
self.calib = OmniCalibration(calib_folder)
else:
self.calib_file = os.path.join(folder_path, 'calib', self.seq_name+'.txt')
self.calib = Calibration(self.calib_file)
self.depth_files = sorted(glob.glob('%s/*.*' % os.path.join(folder_path, 'depth')))
self.depth_files = [file for file in self.depth_files if file.split('.')[-1]=='bin']
else:
self.calib = None
self.cuda = cuda
def __getitem__(self, index):
img_path = self.files[index % len(self.files)]
if self.point_cloud:
depth_path = self.depth_files[index % len(self.depth_files)]
# Extract image
img = np.array(Image.open(img_path))
# Channels-first
input_img = np.transpose(img, (2, 0, 1))/255
# As pytorch tensor
input_img = torch.from_numpy(input_img).float()
if self.cuda:
input_img = input_img.cuda()
frame_idx = int(os.path.basename(img_path)[:-4])
if self.point_cloud:
#velodyne coordinates and image coordinates are different.
#velo_x = camera_z
#velo_y = -camera_x
#velo_z = -camera_y
if self.omni:
scan = np.load(depth_path)
else:
scan = np.fromfile(depth_path, dtype=np.float32)
scan = scan.reshape((-1, 4))
scan[:, :3] = self.calib.project_velo_to_ref(scan[:, :3])
return frame_idx, img_path, input_img, scan
else:
return frame_idx, img_path, input_img, -1
def __len__(self):
return len(self.files)
def is_image_file(file):
IMG_FILE_FORMATS = ['jpg', 'png', 'tif', 'bmp', 'jpeg']
if file.split('.')[-1] in IMG_FILE_FORMATS:
return True
else:
return False
class TripletDataset(Dataset):
def __init__(self, feature_path, num_negative_samples = 100, cuda = True, sequence = False, test = False):
if test:
feature_file = os.path.join(feature_path, 'test_features.npy')
else:
feature_file = os.path.join(feature_path, 'features.npy')
feature_array = np.load(feature_file)
# feature_array = feature_array[:500]
# feature_array = np.vstack([feature_array[146], feature_array[148], feature_array[149],feature_array[32], feature_array[10], feature_array[7],feature_array[9],feature_array[31], feature_array[8]])
# feature_array = np.vstack([feature_array[10], feature_array[11],feature_array[9],feature_array[8],feature_array[249],feature_array[247]])
# self.ids = feature_array[:, 0]
# if not test:
# feature_array = feature_array[self.ids < 5]
self.ids = feature_array[:, 0].astype(np.float32).astype(np.int32)
self.unique_ids = np.unique(self.ids)
self.frames = feature_array[:, 2].astype(np.float32).astype(np.int32)
self.features = feature_array[:, 3:].astype(np.float32)
self.sequences = feature_array[:, 1].astype(np.float32).astype(np.int32)
self.sequence = sequence
if self.sequence:
self.size = self.unique_ids.size
else:
self.size = self.ids.size
self.num_negative_samples = num_negative_samples
self.tensor_type = torch.cuda.FloatTensor if cuda else torch.FloatTensor
def __getitem__(self, index):
if self.sequence:
object_id = self.unique_ids[index]
positive_ids = self.ids == object_id
object_sequence = self.sequences[positive_ids][0]
object_frames = self.frames[positive_ids]
positive_sequence = self.features[positive_ids]
positive_sequence = torch.Tensor(positive_sequence).type(self.tensor_type)
negative_sequence = []
for frame in object_frames[1:]:
idx = np.logical_and(self.sequences==object_sequence, self.frames==frame)
idx = np.logical_and(idx, self.ids!=object_id)
if np.sum(idx)==0:
negative_sequence.append(None)
else:
negative_sequence.append(torch.Tensor(self.features[idx]).type(self.tensor_type))
negative_ids = np.random.choice(len(self.ids), size = self.num_negative_samples, replace = False)
negative_ids = negative_ids[self.ids[negative_ids] != object_id]
negative_features = self.features[negative_ids]
negative_features = torch.Tensor(negative_features).type(self.tensor_type)
return positive_sequence, negative_sequence, negative_features
else:
object_id = self.ids[index]
anchor_feature = self.features[index]
anchor_feature = torch.Tensor(anchor_feature).type(self.tensor_type)
positive_ids = np.where(self.ids == object_id)[0]
positive_feature = self.features[random.choice(positive_ids)]
positive_feature = torch.Tensor(positive_feature).type(self.tensor_type)
negative_ids = np.random.choice(len(self.ids), size = self.num_negative_samples, replace = False)
negative_ids = negative_ids[self.ids[negative_ids] != object_id]
negative_features = self.features[negative_ids]
negative_features = torch.Tensor(negative_features).type(self.tensor_type)
return anchor_feature, positive_feature, negative_features
def __len__(self):
return self.size
class STIPDataset(Dataset):
def __init__(self, folder_path, img_size=416, point_cloud = False, pad = False):
self.files = sorted(glob.glob('%s/imgs/*/*.*' % folder_path), key = lambda x: int(os.path.splitext(os.path.basename(x))[0]))
self.files = [file for file in self.files if is_image_file(file)]
self.img_shape = (img_size, img_size)
self.pad = pad
self.seq_name = os.path.split(folder_path)[-1]
def __getitem__(self, index):
img_path = self.files[index % len(self.files)]
# Extract image
img = np.array(Image.open(img_path))
h, w, _ = img.shape
dim_diff = np.abs(h - w)
# Upper (left) and lower (right) padding
pad1, pad2 = dim_diff // 2, dim_diff - dim_diff // 2
# Determine padding
pad = ((pad1, pad2), (0, 0), (0, 0)) if h <= w else ((0, 0), (pad1, pad2), (0, 0))
# Add padding
if self.pad:
img = np.pad(img, pad, 'constant', constant_values=127.5) / 255.
# Resize and normalize
img = resize(img, (*self.img_shape, 3), mode='reflect', anti_aliasing = True)
# Channels-first
input_img = np.transpose(img, (2, 0, 1))/255
# As pytorch tensor
input_img = torch.from_numpy(input_img).float()
return img_path, input_img, -1
def __len__(self):
return len(self.files)
def collate_fn(inputs):
#ASSUMES BATCH SIZE IS ALWAYS 1
return inputs[0]
================================================
FILE: paper_experiments/utils/deep_sort_utils.py
================================================
# vim: expandtab:ts=4:sw=4
import numpy as np
import cv2
def non_max_suppression(boxes, max_bbox_overlap, scores=None):
"""Suppress overlapping detections.
Original code from [1]_ has been adapted to include confidence score.
.. [1] http://www.pyimagesearch.com/2015/02/16/
faster-non-maximum-suppression-python/
Examples
--------
>>> boxes = [d.roi for d in detections]
>>> scores = [d.confidence for d in detections]
>>> indices = non_max_suppression(boxes, max_bbox_overlap, scores)
>>> detections = [detections[i] for i in indices]
Parameters
----------
boxes : ndarray
Array of ROIs (x, y, width, height).
max_bbox_overlap : float
ROIs that overlap more than this values are suppressed.
scores : Optional[array_like]
Detector confidence score.
Returns
-------
List[int]
Returns indices of detections that have survived non-maxima suppression.
"""
if len(boxes) == 0:
return []
boxes = boxes.astype(np.float)
pick = []
x1 = boxes[:, 0]
y1 = boxes[:, 1]
x2 = boxes[:, 2] + boxes[:, 0]
y2 = boxes[:, 3] + boxes[:, 1]
area = (x2 - x1 + 1) * (y2 - y1 + 1)
if scores is not None:
idxs = np.argsort(scores)
else:
idxs = np.argsort(y2)
while len(idxs) > 0:
last = len(idxs) - 1
i = idxs[last]
pick.append(i)
xx1 = np.maximum(x1[i], x1[idxs[:last]])
yy1 = np.maximum(y1[i], y1[idxs[:last]])
xx2 = np.minimum(x2[i], x2[idxs[:last]])
yy2 = np.minimum(y2[i], y2[idxs[:last]])
w = np.maximum(0, xx2 - xx1 + 1)
h = np.maximum(0, yy2 - yy1 + 1)
#overlap = (w * h) / (area[idxs[:last]]) # + area[idxs[last:last+1]] - w * h) #changed from deepsort to sum both areas
overlap = (w * h) / (area[idxs[:last]] + area[idxs[last:last+1]] - w * h) #changed from deepsort to sum both areas
threshold = np.where(y2[i]-y1[i] < 50, max_bbox_overlap-0.1, max_bbox_overlap)
idxs = np.delete(
idxs, np.concatenate(
([last], np.where(overlap > threshold)[0])))
return pick
================================================
FILE: paper_experiments/utils/detection.py
================================================
# vim: expandtab:ts=4:sw=4
import numpy as np
class Detection(object):
"""
This class represents a bounding box detection in a single image.
Parameters
----------
tlwh : array_like
Bounding box in format `(x, y, w, h)`.
confidence : float
Detector confidence score.
feature : array_like
A feature vector that describes the object contained in this image.
Attributes
----------
tlwh : ndarray
Bounding box in format `(top left x, top left y, width, height)`.
confidence : ndarray
Detector confidence score.
feature : ndarray | NoneType
A feature vector that describes the object contained in this image.
"""
def __init__(self, tlwh, box_3d, confidence, appearance_feature, feature):
self.tlwh = np.asarray(tlwh, dtype=np.float)
# Note that detections format is centre of 3D box and dimensions (not bottom face)
self.box_3d = box_3d
if box_3d is not None:
self.box_3d[1] -= box_3d[4]/2
self.box_3d = np.asarray(box_3d, dtype=np.float32)
self.confidence = float(confidence)
self.appearance_feature = np.asarray(appearance_feature, dtype=np.float32)
if feature is not None:
self.feature = np.asarray(feature, dtype = np.float32)
else:
self.feature = None
def to_tlbr(self):
"""Convert bounding box to format `(min x, min y, max x, max y)`, i.e.,
`(top left, bottom right)`.
"""
ret = self.tlwh.copy()
ret[2:] += ret[:2]
return ret
def to_xyah(self):
"""Convert bounding box to format `(center x, center y, aspect ratio,
height)`, where the aspect ratio is `width / height`.
"""
ret = self.tlwh.copy()
ret[:2] += ret[2:] / 2
ret[2] /= ret[3]
return ret
def to_xywh(self):
"""Convert bounding box to format `(center x, center y, aspect ratio,
height)`, where the aspect ratio is `width / height`.
"""
ret = self.tlwh.copy()
ret[:2] += ret[2:] / 2
return ret
def get_3d_distance(self):
if self.box_3d is not None:
return np.sqrt(self.box_3d[0]**2 + self.box_3d[2]**2)
================================================
FILE: paper_experiments/utils/double_measurement_kf.py
================================================
import random
import numpy as np
import scipy.linalg
import EKF
import pdb
import kf_2d
import os
import pickle
import torch
from copy import deepcopy
import matplotlib.pyplot as plt
from read_detections import read_ground_truth_3d_detections, read_ground_truth_2d_detections
np.set_printoptions(precision=4, suppress=True)
from calibration import Calibration
import sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from evaluation.distances import iou_matrix
class KF_3D(kf_2d.KalmanFilter2D):
"""
3D Kalman Filter that tracks objets in 3D space
The 8-dimensional state space
x, y, z, l, h, w, theta, vx, vz
contains the bounding box center position (x, z), the heading angle theta, the
box dimensions l, w, h, and the x and z velocities.
Object motion follows a constant velocity model. The bounding box location
(x, y) is taken as direct observation of the state space (linear
observation model).
"""
def __init__(self, calib, pos_weight_3d, pos_weight, velocity_weight, theta_weight,
std_process, std_measurement_2d, std_measurement_3d,
initial_uncertainty, omni = True, debug=True):
self.ndim, self.dt = 9, 1.
# Create Kalman filter model matrices.
# Motion model is constant velocity, i.e. x = x + Vx*dt
self._motion_mat = np.eye(self.ndim, self.ndim)
self._motion_mat[0, 7] = self.dt
self._motion_mat[2, 8] = self.dt
# Sensor model is direct observation, i.e. x = x
self._observation_mat = np.eye(self.ndim - 2, self.ndim)
if omni:
self.x_constant = calib.img_shape[2]/(2*np.pi)
self.y_constant = calib.median_focal_length_y
self.calib = calib
else:
self.projection_matrix = calib.P
self.omni = omni
self._std_weight_pos_3d = pos_weight_3d
self._std_weight_pos = pos_weight
self._std_weight_vel = velocity_weight
self._std_weight_theta= theta_weight
self._std_weight_process = std_process
self._initial_uncertainty = initial_uncertainty
self._std_weight_measurement_2d = std_measurement_2d
self._std_weight_measurement_3d = std_measurement_3d
self.debug = debug
def initiate(self, measurement_3d):
mean_pos = measurement_3d
mean_vel = np.zeros((2,))
mean = np.r_[mean_pos, mean_vel]
std = [
self._std_weight_pos_3d * measurement_3d[0],
self._std_weight_pos_3d * measurement_3d[1],
self._std_weight_pos_3d * measurement_3d[2],
self._std_weight_pos_3d * measurement_3d[3],
self._std_weight_pos_3d * measurement_3d[4],
self._std_weight_pos_3d * measurement_3d[5],
self._std_weight_theta,
self._std_weight_vel,
self._std_weight_vel]
covariance = np.diag(np.square(std))*(self._initial_uncertainty*self._std_weight_process)**2
return mean, covariance
def get_process_noise(self, mean):
std_pos = [
self._std_weight_pos_3d, # x
self._std_weight_pos_3d, # y
self._std_weight_pos_3d, # z
self._std_weight_pos_3d, # l
self._std_weight_pos_3d, # h
self._std_weight_pos_3d, # w
self._std_weight_theta # theta
]
std_vel = [
self._std_weight_vel, # x
self._std_weight_vel, # z
]
self._motion_cov = np.diag(np.square(np.r_[std_pos, std_vel]))
motion_cov = np.diag(np.square(np.r_[std_pos, std_vel]))*self._std_weight_process**2
return motion_cov
def get_2d_measurement_noise(self, measurement_2d):
# Returns Qt the sensor noise covariance
# Measurement uncertainty scaled by estimated height
std = [
self._std_weight_pos*measurement_2d[2],
self._std_weight_pos*measurement_2d[3],
self._std_weight_pos*measurement_2d[2],
self._std_weight_pos*measurement_2d[3]]
innovation_cov = np.diag(np.square(std))*self._std_weight_measurement_2d**2
return innovation_cov
def get_3d_measurement_noise(self, measurement):
# Returns Qt the sensor noise covariance
# Measurement uncertainty scaled by estimated height
std = [
self._std_weight_pos_3d * measurement[0], # x
self._std_weight_pos_3d * measurement[1], # y
self._std_weight_pos_3d * measurement[2], # z
self._std_weight_pos_3d * measurement[3], # l
self._std_weight_pos_3d * measurement[4], # h
self._std_weight_pos_3d * measurement[5], # w
self._std_weight_theta # theta
]
innovation_cov = np.diag(np.square(std))*self._std_weight_measurement_3d**2
return innovation_cov
def gating_distance(self, mean, covariance, measurements,
only_position=False,
use_3d=True):
"""Compute gating distance between state distribution and measurements.
A suitable distance threshold can be obtained from `chi2inv95`. If
`only_position` is False, the chi-square distribution has 4 degrees of
freedom, otherwise 2.
Parameters
----------
mean : ndarray
Mean vector over the state distribution (8 dimensional).
covariance : ndarray
Covariance of the state distribution (8x8 dimensional).
measurements : ndarray
An Nx4 dimensional matrix of N measurements, each in
format (x, y, a, h) where (x, y) is the bounding box center
position, a the aspect ratio, and h the height.
only_position : Optional[bool]
If True, distance computation is done with respect to the bounding
box center position only.
Returns
-------
ndarray
Returns an array of length N, where the i-th element contains the
squared Mahalanobis distance between (mean, covariance) and
`measurements[i]`.
"""
if not use_3d:
corner_points, corner_points_3d = self.calculate_corners(mean)
H_2d = self.get_2d_measurement_matrix(mean, corner_points, corner_points_3d)
min_x, min_y = np.amin(corner_points, axis = 0)[:2]
max_x, max_y = np.amax(corner_points, axis = 0)[:2]
cov = self.project_cov_2d(mean, covariance, H_2d)
mean = np.array([min_x, min_y, max_x - min_x, max_y - min_y])
else:
mean, cov = mean[:7], covariance[:7, :7]
if only_position:
if use_3d:
mean, cov = mean[:3], cov[:3, :3]
measurements = measurements[:, :3]
else:
mean, cov = mean[:2], cov[:2, :2]
measurements = measurements[:, :2]
return EKF.squared_mahalanobis_distance(mean, cov, measurements)
def project_cov(self, mean, covariance):
# Returns S the innovation covariance (projected covariance)
measurement_noise = self.get_3d_measurement_noise(mean)
innovation_cov = (np.linalg.multi_dot((self._observation_mat, covariance,
self._observation_mat.T))
+ measurement_noise)
return innovation_cov
def project_cov_2d(self, mean, covariance, H_2d):
# Returns S the innovation covariance (projected covariance)
measurement_noise = self.get_2d_measurement_noise(mean)
innovation_cov = (np.linalg.multi_dot((H_2d, covariance,
H_2d.T))
+ measurement_noise)
return innovation_cov
# @profile
def update(self, mean, covariance, measurement_2d, measurement_3d = None, marginalization=None, JPDA=False):
"""Run Kalman filter correction step.
Parameters
----------
mean : ndarray
The predicted state's mean vector (9 dimensional).
covariance : ndarray
The state's covariance matrix (9x9 dimensional).
measurement_2d : ndarray
The 4 dimensional measurement vector (x, y, w, h), where (x, y)
is the center position, a the aspect ratio, and h the height of the
bounding box.
measurement_3d : ndarray
The 7 dimensional measurement vector (x, y, z, l, h, w, theta), where (x, y, z)
is the center bottom of the box, l, q, h are the dimensions of the bounding box
theta is the orientation angle w.r.t. the positive x axis.
Returns
-------
(ndarray, ndarray)
Returns the measurement-corrected state distribution.
"""
if np.any(np.isnan(mean)):
return mean, covariance
out_cov = covariance
H_3d = self._observation_mat
do_3d = True
covariance_3d = None
for meas in measurement_3d:
if meas is None:
do_3d = False
break
if do_3d:
S_matrix = self.project_cov(mean, out_cov)
try:
chol_factor, lower = scipy.linalg.cho_factor(
S_matrix, lower=True, check_finite=False)
kalman_gain = scipy.linalg.cho_solve(
(chol_factor, lower), np.dot(out_cov, H_3d.T).T,
check_finite=False).T
except:
# in case cholesky factorization fails, revert to standard solver
kalman_gain = np.linalg.multi_dot((out_cov, H_3d.T, np.linalg.inv(S_matrix)))
out_cov -= np.linalg.multi_dot((kalman_gain, S_matrix, kalman_gain.T))
if JPDA:
innovation_3d = 0
cov_uncertainty_3d = 0
for i, detection_3d in enumerate(measurement_3d):
innovation_partial = detection_3d - mean[:7]
innovation_3d += innovation_partial * marginalization[i+1]
cov_uncertainty_3d += marginalization[i+1] * np.outer(innovation_partial, innovation_partial)
partial_cov = cov_uncertainty_3d-np.outer(innovation_3d, innovation_3d)
out_cov *= 1 - marginalization[0]
out_cov += np.linalg.multi_dot((kalman_gain, partial_cov, kalman_gain.T))
out_cov += marginalization[0]*covariance
else:
out_cov = out_cov - np.linalg.multi_dot((kalman_gain, H_3d, out_cov))
innovation_3d = measurement_3d - mean[:7]
mean = mean + np.dot(kalman_gain, innovation_3d)
post_3d_mean = mean
covariance_3d = out_cov
if measurement_2d is not None:
corner_points, corner_points_3d = self.calculate_corners(mean)
H_2d = self.get_2d_measurement_matrix(mean, corner_points, corner_points_3d)
#update based on 2D
min_x, min_y = np.amin(corner_points, axis = 0)[:2]
max_x, max_y = np.amax(corner_points, axis = 0)[:2]
S_matrix = self.project_cov_2d(np.array([min_x, min_y, max_x - min_x, max_y - min_y]), out_cov, H_2d)
try:
chol_factor, lower = scipy.linalg.cho_factor(
S_matrix, lower=True, check_finite=False)
kalman_gain = scipy.linalg.cho_solve(
(chol_factor, lower), np.dot(out_cov, H_2d.T).T,
check_finite=False).T
except:
# in case cholesky factorization fails, revert to standard solver
kalman_gain = np.linalg.multi_dot((out_cov, H_2d.T, np.linalg.inv(S_matrix)))
out_cov = np.dot(np.eye(*out_cov.shape)-np.dot(kalman_gain, H_2d), out_cov)
if JPDA:
innovation_2d = 0
cov_uncertainty_2d = 0
for i, detection_2d in enumerate(measurement_2d):
innovation_partial = detection_2d[:4] - np.array([min_x, min_y, max_x - min_x, max_y - min_y])
innovation_2d += innovation_partial * marginalization[i+1] # +1 to account for dummy node
cov_uncertainty_2d += marginalization[i+1] * np.outer(innovation_partial, innovation_partial)
partial_cov = cov_uncertainty_2d-np.outer(innovation_2d, innovation_2d)
out_cov *= 1 - marginalization[0]
out_cov += np.linalg.multi_dot((kalman_gain, partial_cov, kalman_gain.T))
if covariance_3d is None:
out_cov += marginalization[0]*covariance
else:
out_cov += marginalization[0]*covariance_3d
else:
innovation_2d = measurement_2d[:4] - np.array([min_x, min_y, max_x - min_x, max_y - min_y])
mean = mean + np.dot(kalman_gain, innovation_2d)
if self.debug:
return mean, out_cov, post_3d_mean
return mean, out_cov
# @profile
def get_2d_measurement_matrix(self, mean, corner_points, corner_points_3d):
min_x = np.inf
min_x_idx = None
max_x = -np.inf
max_x_idx = None
min_y = np.inf
min_y_idx = None
max_y = -np.inf
max_y_idx = None
for idx, pt in enumerate(corner_points):
if pt[0] < min_x:
min_x_idx = idx
min_x = pt[0]
if pt[0] > max_x:
max_x_idx = idx
max_x = pt[0]
if pt[1] < min_y:
min_y_idx = idx
min_y = pt[1]
if pt[1] > max_y:
max_y_idx = idx
max_y = pt[1]
if self.omni:
jac_x = np.dot(self.jacobian_omni(corner_points_3d[min_x_idx])[0], self.corner_jacobian(mean, min_x_idx))
jac_y = np.dot(self.jacobian_omni(corner_points_3d[min_y_idx])[1], self.corner_jacobian(mean, min_y_idx))
jac_w = np.dot(self.jacobian_omni(corner_points_3d[max_x_idx])[0], self.corner_jacobian(mean, max_x_idx)) - jac_x
jac_h = np.dot(self.jacobian_omni(corner_points_3d[max_y_idx])[1], self.corner_jacobian(mean, max_y_idx)) - jac_y
else:
jac_x = np.dot(self.jacobian(corner_points_3d[min_x_idx])[0], self.corner_jacobian(mean, min_x_idx))
jac_y = np.dot(self.jacobian(corner_points_3d[min_y_idx])[1], self.corner_jacobian(mean, min_y_idx))
jac_w = np.dot(self.jacobian(corner_points_3d[max_x_idx])[0], self.corner_jacobian(mean, max_x_idx)) - jac_x
jac_h = np.dot(self.jacobian(corner_points_3d[max_y_idx])[1], self.corner_jacobian(mean, max_y_idx)) - jac_y
jac = np.vstack([jac_x, jac_y, jac_w, jac_h])
jac = np.hstack([jac, np.zeros((jac.shape[0], 2))])
return jac
# Jacobian for projective transformation
def jacobian(self, pt_3d):
den = np.sum(self.projection_matrix[2] * pt_3d)
dxy = (1 - self.projection_matrix[2] * pt_3d/den) * self.projection_matrix[0:2]/den
return dxy[:, :3]
def jacobian_omni(self, pt_3d):
jac = np.zeros((2, 3))
x, y, z = pt_3d[0], pt_3d[1], pt_3d[2]
denominator = (x**2 + z**2)
jac[0, 0] = -self.x_constant*(2*x*(z**2)/denominator)
jac[0, 0] /= denominator
jac[0, 2] = self.x_constant*2*z/denominator
jac[0, 2] *= 1 - (z**2)/denominator
jac[1, 0] = self.y_constant*x*y/denominator
jac[1, 1] = -self.y_constant
jac[1,2] = self.y_constant*z*y/denominator
jac[1, :] /= np.sqrt(denominator)
return jac
def calculate_corners(self, box):
x,y,z,l,h,w,theta = box[:7]
pt_3d = []
x_delta_1 = np.cos(theta)*l/2+np.sin(theta)*w/2
x_delta_2 = np.cos(theta)*l/2 - np.sin(theta)*w/2
z_delta_1 = np.sin(theta)*l/2-np.cos(theta)*w/2
z_delta_2 = np.sin(theta)*l/2+np.cos(theta)*w/2
pt_3d.append((x+x_delta_1, y + h/2, z+z_delta_1, 1))
pt_3d.append((x+x_delta_2, y + h/2, z+z_delta_2, 1))
pt_3d.append((x-x_delta_2, y + h/2, z-z_delta_2, 1))
pt_3d.append((x-x_delta_1, y + h/2, z-z_delta_1, 1))
pt_3d.append((x+x_delta_1, y - h/2, z+z_delta_1, 1))
pt_3d.append((x+x_delta_2, y - h/2, z+z_delta_2, 1))
pt_3d.append((x-x_delta_2, y - h/2, z-z_delta_2, 1))
pt_3d.append((x-x_delta_1, y - h/2, z-z_delta_1, 1))
pts_3d = np.vstack(pt_3d)
pts_2d = self.project_2d(pts_3d)
return pts_2d, pts_3d
def corner_jacobian(self, pt_3d, corner_idx):
_, _, _, l, _, w, theta = pt_3d[:7]
jac = np.eye(3,7)
jac[1, 4] = 0.5 if corner_idx < 4 else -0.5
jac[0, 3] = 0.5*np.sin(theta) if corner_idx % 4 < 2 else -0.5*np.sin(theta)
jac[0, 5] = 0.5*np.cos(theta) if corner_idx % 2 == 0 else -0.5*np.cos(theta)
jac[2, 3] = 0.5*np.cos(theta) if corner_idx%4 < 2 else -0.5*np.cos(theta)
jac[2, 5] = 0.5*np.sin(theta) if corner_idx%2 == 0 else -0.5*np.sin(theta)
if corner_idx%4 == 0:
jac[0, 6] = -np.sin(theta)*l/2 + np.cos(theta)*w/2
jac[2, 6] = np.cos(theta)*l/2 + np.sin(theta)*w/2
elif corner_idx%4==1:
jac[0, 6] = -np.sin(theta)*l/2 - np.cos(theta)*w/2
jac[2, 6] = np.cos(theta)*l/2 - np.sin(theta)*w/2
elif corner_idx%4==2:
jac[0, 6] = +np.sin(theta)*l/2 + np.cos(theta)*w/2
jac[2, 6] = -np.cos(theta)*l/2 + np.sin(theta)*w/2
else:
jac[0, 6] = +np.sin(theta)*l/2 - np.cos(theta)*w/2
jac[2, 6] = -np.cos(theta)*l/2 - np.sin(theta)*w/2
return jac
def project_2d(self, pts_3d):
if self.omni:
pts_2d = np.array(self.calib.project_ref_to_image_torch(torch.from_numpy(pts_3d)))
else:
pts_2d = np.dot(pts_3d, self.projection_matrix.T)
pts_2d /= np.expand_dims(pts_2d[:, 2], 1)
return pts_2d[:, :2]
def swap(detections_3d, iou, idx, swap_prob = 0):
if random.random() > swap_prob:
return detections_3d[idx]
else:
iou_row = iou[idx]
iou_row[idx] = -1
max_idx = np.argmax(iou_row)
if iou_row[max_idx] > 0.4:
# print("SWAP")
return detections_3d[max_idx]
else:
return detections_3d[idx]
if __name__ == '__main__':
seq = '0001'
gt_path = os.path.join('data','KITTI','sequences', seq, 'gt')
prob_3d_list = [0.6]
prob_2d_list = [0.9]
swap_prob = 0
std_3d = 0.2
std_2d = 5
boxes_3d, ids, frame_3d = read_ground_truth_3d_detections(os.path.join(gt_path, '3d_detections.txt'), None)
boxes_2d, object_ids, frame_2d = read_ground_truth_2d_detections(os.path.join(gt_path, 'gt.txt'), None, nms_threshold = 1)
boxes_2d[:,2] -= boxes_2d[:,0]
boxes_2d[:,3] -= boxes_2d[:,1]
boxes_3d[:,1] -= boxes_3d[:, 4]/2
calib = Calibration(os.path.join(os.path.dirname(gt_path), 'calib', seq+'.txt'))
pos_weight = 0.05
pos_weight_2d = 0.006
velocity_weight = 0.0007
theta_weight = 0.000300
std_process = 2
std_measurement_2d = 2.6
std_measurement_3d = 0.01
initial_uncertainty = 1
kf = KF_3D(calib, pos_weight, pos_weight_2d, velocity_weight, theta_weight,
std_process, std_measurement_2d, std_measurement_3d,
initial_uncertainty, omni=False, debug=True)
final_errors = np.zeros((len(prob_2d_list), len(prob_3d_list)))
random.seed(14295)
np.random.seed(14295)
for idx_3d, prob_3d in enumerate(prob_3d_list):
for idx_2d, prob_2d in enumerate(prob_2d_list):
id_means = {idx:[] for idx in np.unique(ids)}
id_means_2d = {idx:[] for idx in np.unique(ids)}
id_preds = {idx:[] for idx in np.unique(ids)}
id_meas = {idx:[] for idx in np.unique(ids)}
id_errors = {idx:[] for idx in np.unique(ids)}
for frame in sorted(np.unique(frame_2d)):
frame_mask = frame_2d==frame
frame_boxes_2d = boxes_2d[frame_mask]
frame_boxes_3d = boxes_3d[frame_mask]
frame_ids = ids[frame_mask]
iou = 1-iou_matrix(frame_boxes_2d[:,:4], frame_boxes_2d[:,:4], max_iou=10) #output of function is 1 - IoU
for idx, object_id in enumerate(frame_ids):
if frame_boxes_3d[idx][2] > 30:
continue
noise_2d = np.random.randn(*frame_boxes_2d[idx].shape)*std_2d
noise_3d = np.random.randn(*frame_boxes_3d[idx].shape)*std_3d
if len(id_means[object_id.item()]) == 0:
mean, cov = kf.initiate(frame_boxes_2d[idx]+noise_2d, frame_boxes_3d[idx]+noise_3d)
id_means[object_id.item()].append((mean, cov, frame))
# id_preds[object_id.item()].append((mean, cov, frame))
# id_meas[object_id.item()].append((frame_boxes_3d[idx], frame_boxes_2d[idx], frame))
# id_errors[object_id.item()].append((np.sqrt(np.sum((mean[:3] - frame_boxes_3d[idx][:3])**2)), frame))
continue
mean, cov = kf.predict(id_means[object_id.item()][-1][0], id_means[object_id.item()][-1][1])
id_preds[object_id.item()].append((mean, cov, frame))
# if object_id.item()==3:
# print("3D box: ", frame_boxes_3d[idx])
# print("Old mean:", id_means[object_id.item()][0])
# print("Predicted mean:", mean)
# pdb.set_trace()
if random.random() < prob_2d:
if random.random() < prob_3d:
mean, cov, mean_2d = kf.update(mean, cov, frame_boxes_2d[idx]+noise_2d, swap(frame_boxes_3d, iou, idx, swap_prob)+noise_3d)
else:
mean, cov, mean_2d = kf.update(mean, cov, frame_boxes_2d[idx]+noise_2d, None)
# if object_id.item()==12:
# print("Updated mean after 2D:", mean_2d)
# print("Updated mean after 3D:", mean)
# print("Error:", np.sqrt(np.sum((mean[:3] - frame_boxes_3d[idx][:3])**2)))
# if np.sqrt(np.sum((mean[:3] - frame_boxes_3d[idx][:3])**2)) > 1:
# pdb.set_trace()
id_means[object_id.item()].append((mean, cov, frame))
id_means_2d[object_id.item()].append((mean_2d, frame))
id_meas[object_id.item()].append((frame_boxes_3d[idx], frame_boxes_2d[idx], frame))
id_errors[object_id.item()].append((np.sqrt(np.sum((mean[:3] - frame_boxes_3d[idx][:3])**2)), frame))
errors = [np.mean(error[0]) for idx, error in id_errors.items() if len(error) > 0]
final_errors[idx_2d, idx_3d] = np.mean(errors)
print("3D prob: %f %% & 2D prob: %f %% & swap prob: %f %% RMSE: %f"%(prob_3d*100, prob_2d*100, swap_prob*100, final_errors[idx_2d, idx_3d]))
# if :
with open('results/kf_mean_pickle.p', 'wb') as f:
pickle.dump([id_means, id_means_2d, id_meas, id_preds], f)
print(final_errors)
================================================
FILE: paper_experiments/utils/evaluate_detections.py
================================================
import numpy as np
import os
import pdb
from tqdm import tqdm
from deep_sort_utils import non_max_suppression as deepsort_nms
from visualise import draw_track
import matplotlib.pyplot as plt
from PIL import Image
def evaluate_detections(detection_path_1, detection_path_2, detection_path_3, detection_path_4, gt_path):
#expecting detections and gt in file with format as in read_detections.py
# applies confidence thresholding
try:
detections_1 = np.loadtxt(detection_path_1, delimiter=',')
# detections_2 = np.loadtxt(detection_path_2, delimiter=',')
# detections_3 = np.loadtxt(detection_path_3, delimiter=',')
# detections_4 = np.loadtxt(detection_path_4, delimiter=',')
# detections = np.concatenate([detections_1, detections_2, detections_3, detections_4])
detections = detections_1
gt = np.loadtxt(gt_path, delimiter=',')
except:
return
gt_frames = gt[:, 0]
det_confidence = detections[:, 6]
###CONFIDENCE THRESHOLD
detections = detections[det_confidence > 0.9]
########
print("Average number of detections per frame = %f"%(detections.shape[0]/len(np.unique(gt_frames))))
det_frames = detections[:, 0]
det_confidence = detections[:, 6]
gt_boxes = np.asarray(list(zip(gt[:, 2], gt[:, 3], gt[:, 4], gt[:, 5])))
det_boxes = np.asarray(list(zip(detections[:, 2], detections[:, 3], detections[:, 4], detections[:, 5])))
assignments = []
missed_detections = 0
for frame in np.unique(gt_frames):
frame_mask_det = det_frames == frame
frame_mask_gt = gt_frames == frame
frame_gt_boxes = gt_boxes[frame_mask_gt]
frame_det_boxes = det_boxes[frame_mask_det]
frame_confidence = det_confidence[frame_mask_det]
x1 = np.expand_dims(detections[frame_mask_det,2].astype(np.float32), 1)
y1 = np.expand_dims(detections[frame_mask_det,3].astype(np.float32), 1)
w = np.expand_dims(detections[frame_mask_det,4].astype(np.float32), 1)
h = np.expand_dims(detections[frame_mask_det,5].astype(np.float32), 1)
conf = np.expand_dims(detections[frame_mask_det,6].astype(np.float32), 1)
boxes = np.hstack([x1, y1, w, h])
indices = deepsort_nms(boxes, 0.75, np.squeeze(conf))
frame_det_boxes = frame_det_boxes[indices]
# print(frame_confidence)
positive_arr = np.asarray([False]*len(frame_det_boxes))
for i, gt_box in enumerate(frame_gt_boxes):
iou_list = np.asarray([iou(gt_box, det_box) for det_box in frame_det_boxes])
positive_idx = np.where(iou_list >= 0.5)[0]
if len(positive_idx) == 0:
missed_detections += 1
plt.figure(0)
plt.imshow(Image.open(os.path.join(os.path.split(detection_path_1)[0], '..','imgs','%.6d.png'%frame)))
draw_track(None, gt_box, det = False)
for det_box in frame_det_boxes:
draw_track(None, det_box, det = True)
# print(det_box)
# print('Boxes:')
# print(boxes)
# print('FRAME DONE')
plt.show()
positive_arr[positive_idx] = True
assignments.extend(list(zip(positive_arr, frame_confidence)))
assignments = sorted(assignments, key = lambda x: x[1], reverse = True)
predictions = list(zip(*assignments))[0]
true_positives = np.cumsum(predictions)
false_negatives = np.cumsum(predictions[::-1])[::-1]+missed_detections
precision = true_positives/range(1,len(true_positives)+1)
recall = true_positives/(true_positives + false_negatives)
print("Total missed detections = %d"%missed_detections)
base = 0
idx = []
for i,recall_val in enumerate(recall):
if recall_val > base:
base += 0.1
idx.append(i)
if base >1:
break
precision_vals = [np.amax(precision[index:]) for index in idx]
if len(precision_vals) < 11:
precision_vals.extend([0]*(11-len(precision_vals)))
print(precision_vals)
return np.mean(precision_vals)
def iou(bbox_1, bbox_2):
x1_1, y1_1, w_1, h_1 = bbox_1
x1_2, y1_2, w_2, h_2 = bbox_2
x2_1 = x1_1 + w_1
y2_1 = y1_1 + h_1
x2_2 = x1_2 + w_2
y2_2 = y1_2 + h_2
area_1 = abs(x2_1 - x1_1)*abs(y2_1-y1_1)
area_2 = abs(x2_2 - x1_2)*abs(y2_2-y1_2)
intersection = max(0, (min(x2_1, x2_2) - max(x1_1, x1_2))) * max(0, (min(y2_1, y2_2) - max(y1_1, y1_2)))
union = area_1 + area_2 - intersection
return intersection / union
if __name__=='__main__':
ap = []
KITTI_root = 'data/KITTI/sequences'
for sequence in tqdm(range(21)):
ap.append(evaluate_detections(os.path.join(KITTI_root, '%.4d'%sequence, 'det','subcnn_car_det.txt'),
os.path.join(KITTI_root, '%.4d'%sequence, 'det','rrc_car_det.txt'),
os.path.join(KITTI_root, '%.4d'%sequence, 'det','lsvm_car_det.txt'),
os.path.join(KITTI_root, '%.4d'%sequence, 'det','regionlets_car_det.txt'),
os.path.join(KITTI_root, '%.4d'%sequence, 'gt', 'gt_car.txt')))
ap = [ap_val for ap_val in ap if ap_val is not None]
print("FINAL AVERAGE PRECISION OVER ALL SEQUENCES IS: %f"%np.mean(ap))
================================================
FILE: paper_experiments/utils/featurepointnet_model_util.py
================================================
import open3d as o3d
import numpy as np
import tensorflow as tf
import os
import sys
import torch
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.append(BASE_DIR)
import featurepointnet_tf_util as tf_util
# -----------------
# Global Constants
# -----------------
NUM_HEADING_BIN = 12
NUM_SIZE_CLUSTER = 8 # one cluster for each type
NUM_OBJECT_POINT = 512
g_type2class={'Car':0, 'Van':1, 'Truck':2, 'Pedestrian':3,
'Person_sitting':4, 'Cyclist':5, 'Tram':6, 'Misc':7}
g_class2type = {g_type2class[t]:t for t in g_type2class}
g_type2onehotclass = {'Car': 0, 'Pedestrian': 1, 'Cyclist': 2}
#Added 0.5m and 0.2m for car and pedestrian to make boxes slightly bigger
g_type_mean_size = {'Car': np.array([3.88311640418,1.62856739989,1.52563191462]),
'Van': np.array([5.06763659,1.9007158,2.20532825]),
'Truck': np.array([10.13586957,2.58549199,3.2520595]),
'Pedestrian': np.array([0.84422524,0.66068622,1.76255119]),
gitextract_mbelvxqp/
├── CMakeLists.txt
├── LICENSE
├── README.md
├── calib/
│ ├── cameras.yaml
│ └── defaults.yaml
├── config/
│ └── featurepointnet.cfg
├── launch/
│ └── jpda_tracker.launch
├── msg/
│ ├── __init__.py
│ ├── detection2d_with_feature.msg
│ ├── detection2d_with_feature_array.msg
│ ├── detection3d_with_feature.msg
│ └── detection3d_with_feature_array.msg
├── package.xml
├── paper_experiments/
│ ├── models/
│ │ ├── __init__.py
│ │ ├── aligned_reid_model.py
│ │ ├── combination_model.py
│ │ ├── deep_sort_model.py
│ │ ├── featurepointnet_model.py
│ │ ├── pointnet_model.py
│ │ ├── resnet_reid_models.py
│ │ └── yolo_models.py
│ ├── requirements.txt
│ ├── track.py
│ └── utils/
│ ├── EKF.py
│ ├── JPDA_matching.py
│ ├── aligned_reid_utils.py
│ ├── assign_ids_detections.py
│ ├── calibration.py
│ ├── combine_and_process_detections.py
│ ├── dataset.py
│ ├── deep_sort_utils.py
│ ├── detection.py
│ ├── double_measurement_kf.py
│ ├── evaluate_detections.py
│ ├── featurepointnet_model_util.py
│ ├── featurepointnet_tf_util.py
│ ├── imm.py
│ ├── iou_matching.py
│ ├── kf_2d.py
│ ├── kf_3d.py
│ ├── linear_assignment.py
│ ├── logger.py
│ ├── mbest_ilp.py
│ ├── nn_matching.py
│ ├── pointnet_tf_util.py
│ ├── pointnet_transform_nets.py
│ ├── read_detections.py
│ ├── resnet_reid_utils.py
│ ├── test_jpda.py
│ ├── test_kf/
│ │ ├── .gitignore
│ │ ├── run_kf_test.py
│ │ ├── single_track_4state_test.p.val
│ │ ├── single_track_6state_test.p.val
│ │ ├── two_track_4state_test.p.val
│ │ └── write_kf_test.py
│ ├── track.py
│ ├── track_3d.py
│ ├── tracker.py
│ ├── tracker_3d.py
│ ├── tracking_utils.py
│ ├── visualise.py
│ └── yolo_utils/
│ ├── __init__.py
│ ├── datasets.py
│ ├── parse_config.py
│ └── utils.py
├── requirements.txt
└── src/
├── 3d_detector.py
├── EKF.py
├── JPDA_matching.py
├── __init__.py
├── aligned_reid_model.py
├── aligned_reid_utils.py
├── calibration.py
├── combination_model.py
├── deep_sort_utils.py
├── detection.py
├── distances.py
├── double_measurement_kf.py
├── evaluation/
│ ├── __init__.py
│ ├── distances 2.py
│ └── distances.py
├── featurepointnet_model.py
├── featurepointnet_model_util.py
├── featurepointnet_tf_util.py
├── iou_matching.py
├── kf_2d.py
├── linear_assignment.py
├── mbest_ilp.py
├── nn_matching.py
├── pointnet_model.py
├── template 2.py
├── template.py
├── track_3d 2.py
├── track_3d.py
├── tracker_3d 2.py
├── tracker_3d.py
├── tracker_3d_node 2.py
├── tracker_3d_node.py
├── tracking_utils 2.py
└── tracking_utils.py
SYMBOL INDEX (839 symbols across 76 files)
FILE: paper_experiments/models/aligned_reid_model.py
class Model (line 10) | class Model(nn.Module):
method __init__ (line 11) | def __init__(self, local_conv_out_channels=128, num_classes=None):
method forward (line 24) | def forward(self, x):
function conv3x3 (line 60) | def conv3x3(in_planes, out_planes, stride=1):
class BasicBlock (line 66) | class BasicBlock(nn.Module):
method __init__ (line 69) | def __init__(self, inplanes, planes, stride=1, downsample=None):
method forward (line 79) | def forward(self, x):
class Bottleneck (line 98) | class Bottleneck(nn.Module):
method __init__ (line 101) | def __init__(self, inplanes, planes, stride=1, downsample=None):
method forward (line 114) | def forward(self, x):
class ResNet (line 137) | class ResNet(nn.Module):
method __init__ (line 139) | def __init__(self, block, layers):
method _make_layer (line 160) | def _make_layer(self, block, planes, blocks, stride=1):
method forward (line 177) | def forward(self, x):
function remove_fc (line 191) | def remove_fc(state_dict):
function resnet18 (line 200) | def resnet18(pretrained=False):
function resnet34 (line 212) | def resnet34(pretrained=False):
function resnet50 (line 224) | def resnet50(pretrained=False):
function resnet101 (line 236) | def resnet101(pretrained=False):
function resnet152 (line 249) | def resnet152(pretrained=False):
FILE: paper_experiments/models/combination_model.py
class CombiNet (line 6) | class CombiNet(nn.Module):
method __init__ (line 7) | def __init__(self, in_dim = 2560, hidden_units = 512, out_dim = 2560):
method forward (line 16) | def forward(self, x):
class CombiLSTM (line 30) | class CombiLSTM(nn.Module):
method __init__ (line 31) | def __init__(self, in_dim = 2560, hidden_units = 512, out_dim = 2560):
method forward (line 43) | def forward(self, x, hidden = None):
function weight_init (line 65) | def weight_init(m):
FILE: paper_experiments/models/deep_sort_model.py
class ImageEncoder (line 5) | class ImageEncoder(object):
method __init__ (line 7) | def __init__(self, checkpoint_filename="weights/deep_sort_weights.pb",...
method __call__ (line 26) | def __call__(self, data_x):
FILE: paper_experiments/models/featurepointnet_model.py
class FPointNet (line 13) | class FPointNet():
method __init__ (line 14) | def __init__(self, config_path):
method __call__ (line 57) | def __call__(self, input_point_cloud, rot_angle, peds=False):
method get_instance_seg_v1_net (line 130) | def get_instance_seg_v1_net(self, point_cloud, one_hot_vec, is_trainin...
method get_3d_box_estimation_v1_net (line 201) | def get_3d_box_estimation_v1_net(self, object_point_cloud, one_hot_vec...
method get_model (line 250) | def get_model(self, point_cloud, one_hot_vec, is_training, bn_decay=No...
method get_depth_feature_op (line 298) | def get_depth_feature_op(self, is_training):
method get_depth_feature (line 321) | def get_depth_feature(self, object_pointcloud):
method softmax (line 327) | def softmax(self, x):
function create_depth_model (line 334) | def create_depth_model(model, config_path):
FILE: paper_experiments/models/pointnet_model.py
class PointNet (line 9) | class PointNet():
method __init__ (line 10) | def __init__(self, config_path):
method __call__ (line 37) | def __call__(self, input_point_cloud):
method placeholder_inputs (line 43) | def placeholder_inputs(self, batch_size, num_point):
method get_model (line 49) | def get_model(self, point_cloud, is_training, bn_decay=None):
method get_loss (line 96) | def get_loss(self, pred, label, end_points, reg_weight=0.001):
FILE: paper_experiments/models/resnet_reid_models.py
class FeatureResNet (line 8) | class FeatureResNet(nn.Module):
method __init__ (line 9) | def __init__(self,n_layers=50,pretrained=True):
method forward (line 25) | def forward(self,x):
class ResNet (line 30) | class ResNet(nn.Module):
method __init__ (line 31) | def __init__(self,n_id,n_layers=50,pretrained=True):
method forward (line 47) | def forward(self,x):
class NLayersFC (line 54) | class NLayersFC(nn.Module):
method __init__ (line 55) | def __init__(self, in_dim, out_dim, hidden_dim=1, n_layers=0):
method forward (line 69) | def forward(self, x):
class ICT_ResNet (line 72) | class ICT_ResNet(nn.Module):
method __init__ (line 73) | def __init__(self,n_id,n_color,n_type,n_layers=50,pretrained=True):
method forward (line 91) | def forward(self,x):
class TripletNet (line 101) | class TripletNet(nn.Module):
method __init__ (line 102) | def __init__(self, net):
method forward (line 106) | def forward(self, x, y, z):
FILE: paper_experiments/models/yolo_models.py
function create_modules (line 19) | def create_modules(module_defs):
class EmptyLayer (line 94) | class EmptyLayer(nn.Module):
method __init__ (line 97) | def __init__(self):
class Interpolate (line 100) | class Interpolate(nn.Module):
method __init__ (line 101) | def __init__(self, scale_factor, mode):
method forward (line 107) | def forward(self, x):
class YOLOLayer (line 111) | class YOLOLayer(nn.Module):
method __init__ (line 114) | def __init__(self, anchors, num_classes, img_dim):
method forward (line 128) | def forward(self, x, targets=None):
class Darknet (line 240) | class Darknet(nn.Module):
method __init__ (line 243) | def __init__(self, config_path):
method forward (line 252) | def forward(self, x, targets=None):
method load_weights (line 282) | def load_weights(self, weights_path):
method save_weights (line 337) | def save_weights(self, path, cutoff=-1):
FILE: paper_experiments/track.py
function parse_arguments (line 28) | def parse_arguments():
function main (line 92) | def main(opt):
FILE: paper_experiments/utils/EKF.py
function squared_mahalanobis_distance (line 79) | def squared_mahalanobis_distance(mean, covariance, measurements):
class EKF (line 106) | class EKF(object):
method __init__ (line 112) | def __init__(self):
method initiate (line 115) | def initiate(self, measurement):
method predict_mean (line 132) | def predict_mean(self, mean):
method get_process_noise (line 138) | def get_process_noise(self, mean, covariance):
method predict_covariance (line 141) | def predict_covariance(self, mean, covariance):
method project_mean (line 144) | def project_mean(self, mean):
method project_cov (line 149) | def project_cov(self, mean, covariance):
method predict (line 152) | def predict(self, mean, covariance, last_detection, next_to_last_detec...
method get_innovation_cov (line 176) | def get_innovation_cov(self, covariance):
method project (line 179) | def project(self, mean, covariance):
method update (line 200) | def update(self, mean, covariance, measurement_t, marginalization=None...
FILE: paper_experiments/utils/JPDA_matching.py
function get_unmatched (line 8) | def get_unmatched(all_idx, matches, i, marginalization=None):
class Matcher (line 19) | class Matcher:
method __init__ (line 21) | def __init__(self, detections, marginalizations, confirmed_tracks,
method match (line 31) | def match(self):
method get_matches (line 37) | def get_matches(self):
method get_unmatched_tracks (line 51) | def get_unmatched_tracks(self):
method get_unmatched_detections (line 55) | def get_unmatched_detections(self):
method max_match (line 58) | def max_match(self):
method max_and_threshold_matching (line 84) | def max_and_threshold_matching(self):
method hungarian (line 103) | def hungarian(self):
FILE: paper_experiments/utils/aligned_reid_utils.py
function time_str (line 17) | def time_str(fmt=None):
function load_pickle (line 23) | def load_pickle(path):
function save_pickle (line 35) | def save_pickle(obj, path):
function save_mat (line 42) | def save_mat(ndarray, path):
function to_scalar (line 47) | def to_scalar(vt):
function transfer_optim_state (line 58) | def transfer_optim_state(state, device_id=-1):
function may_transfer_optims (line 83) | def may_transfer_optims(optims, device_id=-1):
function may_transfer_modules_optims (line 96) | def may_transfer_modules_optims(modules_and_or_optims, device_id=-1):
class TransferVarTensor (line 115) | class TransferVarTensor(object):
method __init__ (line 118) | def __init__(self, device_id=-1):
method __call__ (line 121) | def __call__(self, var_or_tensor):
class TransferModulesOptims (line 126) | class TransferModulesOptims(object):
method __init__ (line 129) | def __init__(self, device_id=-1):
method __call__ (line 132) | def __call__(self, modules_and_or_optims):
function set_devices (line 136) | def set_devices(sys_device_ids):
function set_devices_for_ml (line 165) | def set_devices_for_ml(sys_device_ids):
function load_ckpt (line 230) | def load_ckpt(modules_optims, ckpt_file, load_to_cpu=True, verbose=True):
function save_ckpt (line 253) | def save_ckpt(modules_optims, ep, scores, ckpt_file):
function load_state_dict (line 274) | def load_state_dict(model, src_state_dict):
function is_iterable (line 315) | def is_iterable(obj):
function may_set_mode (line 319) | def may_set_mode(maybe_modules, mode):
function may_make_dir (line 332) | def may_make_dir(path):
class AverageMeter (line 348) | class AverageMeter(object):
method __init__ (line 352) | def __init__(self):
method reset (line 358) | def reset(self):
method update (line 364) | def update(self, val, n=1):
class RunningAverageMeter (line 371) | class RunningAverageMeter(object):
method __init__ (line 374) | def __init__(self, hist=0.99):
method reset (line 379) | def reset(self):
method update (line 383) | def update(self, val):
class RecentAverageMeter (line 391) | class RecentAverageMeter(object):
method __init__ (line 394) | def __init__(self, hist_size=100):
method reset (line 399) | def reset(self):
method update (line 403) | def update(self, val):
method avg (line 410) | def avg(self):
function get_model_wrapper (line 415) | def get_model_wrapper(model, multi_gpu):
class ReDirectSTD (line 423) | class ReDirectSTD(object):
method __init__ (line 443) | def __init__(self, fpath=None, console='stdout', immediately_visible=F...
method __del__ (line 464) | def __del__(self):
method __enter__ (line 467) | def __enter__(self):
method __exit__ (line 470) | def __exit__(self, *args):
method write (line 473) | def write(self, msg):
method flush (line 485) | def flush(self):
method close (line 492) | def close(self):
function set_seed (line 498) | def set_seed(seed):
function print_array (line 515) | def print_array(array, fmt='{:.2f}', end=' '):
function str2bool (line 526) | def str2bool(v):
function tight_float_str (line 530) | def tight_float_str(x, fmt='{:.4f}'):
function find_index (line 534) | def find_index(seq, item):
function adjust_lr_exp (line 541) | def adjust_lr_exp(optimizer, base_lr, ep, total_ep, start_decay_at_ep):
function adjust_lr_staircase (line 573) | def adjust_lr_staircase(optimizer, base_lr, ep, decay_at_epochs, factor):
function measure_time (line 609) | def measure_time(enter_msg):
function generate_features (line 616) | def generate_features(appearance_model, patches, opt, object_ids = None):
function generate_features_batched (line 645) | def generate_features_batched(appearance_model, patches, opt, object_ids...
function get_image_patches (line 713) | def get_image_patches(input_img, detections):
function create_appearance_model (line 735) | def create_appearance_model(model_type, alignreid_checkpoint, resnet_rei...
FILE: paper_experiments/utils/assign_ids_detections.py
function assign_detection_id (line 11) | def assign_detection_id(detection_path, gt_path, conf_threshold = 0, iou...
FILE: paper_experiments/utils/calibration.py
class Calibration (line 8) | class Calibration(object):
method __init__ (line 33) | def __init__(self, calib_filepath):
method read_calib_file (line 71) | def read_calib_file(self, filepath):
method read_calib_from_video (line 92) | def read_calib_from_video(self, calib_root_dir):
method cart2hom (line 107) | def cart2hom(self, pts_3d):
method cart2hom_torch (line 115) | def cart2hom_torch(self, pts_3d):
method project_velo_to_ref (line 123) | def project_velo_to_ref(self, pts_3d_velo):
method project_ref_to_velo (line 127) | def project_ref_to_velo(self, pts_3d_ref):
method project_rect_to_ref (line 131) | def project_rect_to_ref(self, pts_3d_rect):
method project_ref_to_rect (line 135) | def project_ref_to_rect(self, pts_3d_ref):
method project_ref_to_rect_torch (line 139) | def project_ref_to_rect_torch(self, pts_3d_ref):
method project_rect_to_velo (line 143) | def project_rect_to_velo(self, pts_3d_rect):
method project_velo_to_rect (line 150) | def project_velo_to_rect(self, pts_3d_velo):
method project_rect_to_image (line 157) | def project_rect_to_image(self, pts_3d_rect):
method project_rect_to_image_torch (line 167) | def project_rect_to_image_torch(self, pts_3d_rect):
method project_ref_to_image_torch (line 177) | def project_ref_to_image_torch(self, pts_3d_ref):
method project_velo_to_image (line 187) | def project_velo_to_image(self, pts_3d_velo):
method project_image_to_rect (line 197) | def project_image_to_rect(self, uv_depth):
method project_image_to_velo (line 211) | def project_image_to_velo(self, uv_depth):
function rotx (line 215) | def rotx(t):
function roty (line 224) | def roty(t):
function rotz (line 233) | def rotz(t):
function transform_from_rot_trans (line 242) | def transform_from_rot_trans(R, t):
function inverse_rigid_trans (line 249) | def inverse_rigid_trans(Tr):
function read_label (line 258) | def read_label(label_filename):
function load_image (line 263) | def load_image(img_filename):
function load_velo_scan (line 266) | def load_velo_scan(velo_filename):
function project_to_image (line 271) | def project_to_image(pts_3d, P):
function compute_box_3d (line 291) | def compute_box_3d(obj, P):
function compute_orientation_3d (line 329) | def compute_orientation_3d(obj, P):
function draw_projected_box3d (line 358) | def draw_projected_box3d(image, qs, color=(255,255,255), thickness=2):
class OmniCalibration (line 385) | class OmniCalibration(Calibration):
method __init__ (line 386) | def __init__(self, calib_folder):
method project_ref_to_image_torch (line 402) | def project_ref_to_image_torch(self, pointcloud):
method project_image_to_rect (line 413) | def project_image_to_rect(self, uvdepth):
method project_velo_to_ref (line 422) | def project_velo_to_ref(self, pointcloud):
method move_lidar_to_camera_frame (line 430) | def move_lidar_to_camera_frame(self, pointcloud, upper = True):
method calculate_median_param_value (line 445) | def calculate_median_param_value(self, param):
FILE: paper_experiments/utils/combine_and_process_detections.py
function threshold (line 13) | def threshold(filename, thresh, min, max):
FILE: paper_experiments/utils/dataset.py
class SequenceDataset (line 23) | class SequenceDataset(Dataset):
method __init__ (line 24) | def __init__(self, folder_path, point_cloud=False, cuda=False, omni=Fa...
method __getitem__ (line 44) | def __getitem__(self, index):
method __len__ (line 75) | def __len__(self):
function is_image_file (line 78) | def is_image_file(file):
class TripletDataset (line 85) | class TripletDataset(Dataset):
method __init__ (line 87) | def __init__(self, feature_path, num_negative_samples = 100, cuda = Tr...
method __getitem__ (line 115) | def __getitem__(self, index):
method __len__ (line 157) | def __len__(self):
class STIPDataset (line 160) | class STIPDataset(Dataset):
method __init__ (line 161) | def __init__(self, folder_path, img_size=416, point_cloud = False, pad...
method __getitem__ (line 169) | def __getitem__(self, index):
method __len__ (line 194) | def __len__(self):
function collate_fn (line 197) | def collate_fn(inputs):
FILE: paper_experiments/utils/deep_sort_utils.py
function non_max_suppression (line 6) | def non_max_suppression(boxes, max_bbox_overlap, scores=None):
FILE: paper_experiments/utils/detection.py
class Detection (line 5) | class Detection(object):
method __init__ (line 29) | def __init__(self, tlwh, box_3d, confidence, appearance_feature, featu...
method to_tlbr (line 44) | def to_tlbr(self):
method to_xyah (line 52) | def to_xyah(self):
method to_xywh (line 60) | def to_xywh(self):
method get_3d_distance (line 67) | def get_3d_distance(self):
FILE: paper_experiments/utils/double_measurement_kf.py
class KF_3D (line 19) | class KF_3D(kf_2d.KalmanFilter2D):
method __init__ (line 34) | def __init__(self, calib, pos_weight_3d, pos_weight, velocity_weight, ...
method initiate (line 65) | def initiate(self, measurement_3d):
method get_process_noise (line 84) | def get_process_noise(self, mean):
method get_2d_measurement_noise (line 102) | def get_2d_measurement_noise(self, measurement_2d):
method get_3d_measurement_noise (line 114) | def get_3d_measurement_noise(self, measurement):
method gating_distance (line 130) | def gating_distance(self, mean, covariance, measurements,
method project_cov (line 179) | def project_cov(self, mean, covariance):
method project_cov_2d (line 188) | def project_cov_2d(self, mean, covariance, H_2d):
method update (line 197) | def update(self, mean, covariance, measurement_2d, measurement_3d = No...
method get_2d_measurement_matrix (line 301) | def get_2d_measurement_matrix(self, mean, corner_points, corner_points...
method jacobian (line 338) | def jacobian(self, pt_3d):
method jacobian_omni (line 344) | def jacobian_omni(self, pt_3d):
method calculate_corners (line 360) | def calculate_corners(self, box):
method corner_jacobian (line 379) | def corner_jacobian(self, pt_3d, corner_idx):
method project_2d (line 406) | def project_2d(self, pts_3d):
function swap (line 415) | def swap(detections_3d, iou, idx, swap_prob = 0):
FILE: paper_experiments/utils/evaluate_detections.py
function evaluate_detections (line 11) | def evaluate_detections(detection_path_1, detection_path_2, detection_pa...
function iou (line 99) | def iou(bbox_1, bbox_2):
FILE: paper_experiments/utils/featurepointnet_model_util.py
function tf_gather_object_pc (line 39) | def tf_gather_object_pc(point_cloud, mask, npoints=512):
function get_box3d_corners_helper (line 72) | def get_box3d_corners_helper(centers, headings, sizes):
function get_box3d_corners (line 99) | def get_box3d_corners(center, heading_residuals, size_residuals):
function huber_loss (line 124) | def huber_loss(error, delta):
function parse_output_to_tensors (line 132) | def parse_output_to_tensors(output, end_points):
function from_prediction_to_label_format (line 170) | def from_prediction_to_label_format(center, angle_class, angle_res,\
function size2class (line 179) | def size2class(size, type_name):
function class2size (line 194) | def class2size(pred_cls, residual):
function angle2class (line 199) | def angle2class(angle, num_class):
function class2angle (line 219) | def class2angle(pred_cls, residual, num_class, to_label_format=True):
function rotate_pc_along_y (line 230) | def rotate_pc_along_y(pc, rot_angle):
function placeholder_inputs (line 249) | def placeholder_inputs(batch_size, num_point):
function point_cloud_masking (line 274) | def point_cloud_masking(point_cloud, logits, end_points, xyz_only=True):
function get_center_regression_net (line 322) | def get_center_regression_net(object_point_cloud, one_hot_vec,
function get_loss (line 360) | def get_loss(mask_label, center_label, \
function get_lidar_in_image_fov (line 482) | def get_lidar_in_image_fov(pc_velo, calib, xmin, ymin, xmax, ymax,
function preprocess_pointcloud (line 497) | def preprocess_pointcloud(detections, point_cloud, pc_image_coord,
function generate_detections_3d (line 551) | def generate_detections_3d(detector, detections_2d, point_cloud, calib, ...
function convert_depth_features (line 564) | def convert_depth_features(depth_features_orig, ids_3d, cuda = True):
FILE: paper_experiments/utils/featurepointnet_tf_util.py
function _variable_on_cpu (line 10) | def _variable_on_cpu(name, shape, initializer, use_fp16=False):
function _variable_with_weight_decay (line 24) | def _variable_with_weight_decay(name, shape, stddev, wd, use_xavier=True):
function conv1d (line 52) | def conv1d(inputs,
function conv2d (line 120) | def conv2d(inputs,
function conv2d_transpose (line 188) | def conv2d_transpose(inputs,
function conv3d (line 268) | def conv3d(inputs,
function fully_connected (line 327) | def fully_connected(inputs,
function max_pool2d (line 366) | def max_pool2d(inputs,
function avg_pool2d (line 391) | def avg_pool2d(inputs,
function max_pool3d (line 417) | def max_pool3d(inputs,
function avg_pool3d (line 442) | def avg_pool3d(inputs,
function batch_norm_template_unused (line 468) | def batch_norm_template_unused(inputs, is_training, scope, moments_dims,...
function batch_norm_template (line 512) | def batch_norm_template(inputs, is_training, scope, moments_dims_unused,...
function batch_norm_for_fc (line 534) | def batch_norm_for_fc(inputs, is_training, bn_decay, scope):
function batch_norm_for_conv1d (line 548) | def batch_norm_for_conv1d(inputs, is_training, bn_decay, scope, data_for...
function batch_norm_for_conv2d (line 565) | def batch_norm_for_conv2d(inputs, is_training, bn_decay, scope, data_for...
function batch_norm_for_conv3d (line 580) | def batch_norm_for_conv3d(inputs, is_training, bn_decay, scope):
function dropout (line 594) | def dropout(inputs,
FILE: paper_experiments/utils/imm.py
class IMMFilter2D (line 11) | class IMMFilter2D(EKF.EKF):
method __init__ (line 31) | def __init__(self, kf_vel_params=(1./20, 1./160, 1, 1, 2), kf_walk_par...
method initiate (line 38) | def initiate(self, measurement, flow):
method gating_distance (line 64) | def gating_distance(self, mean, covariance, measurements,
method update (line 94) | def update(self, mean, covariance, measurement, model_probabilities, m...
method predict (line 147) | def predict(self, mean, covariance, model_probabilities):
method mix_models (line 176) | def mix_models(self, mean_1, cov_1, mean_2, cov_2, model_transition_pr...
method combine_states (line 193) | def combine_states(mean, cov, model_probabilities):
function generate_particle_motion (line 200) | def generate_particle_motion(motion_matrices, initial_state, process_noi...
function generate_observations (line 215) | def generate_observations(input_state_list, observation_matrix, observat...
FILE: paper_experiments/utils/iou_matching.py
function iou (line 8) | def iou(bbox, candidates):
function iou_cost (line 43) | def iou_cost(tracks, detections, track_indices=None,
FILE: paper_experiments/utils/kf_2d.py
class KalmanFilter2D (line 8) | class KalmanFilter2D(EKF.EKF):
method __init__ (line 25) | def __init__(self, pos_weight, velocity_weight, std_process, std_measu...
method initiate (line 47) | def initiate(self, measurement, flow):
method predict_mean (line 87) | def predict_mean(self, mean):
method predict_covariance (line 93) | def predict_covariance(self, mean, covariance, last_detection, next_to...
method get_process_noise (line 101) | def get_process_noise(self, mean, last_detection, next_to_last_detecti...
method project_mean (line 136) | def project_mean(self, mean):
method get_measurement_noise (line 144) | def get_measurement_noise(self, measurement):
method project_cov (line 156) | def project_cov(self, mean, covariance):
method gating_distance (line 165) | def gating_distance(self, mean, covariance, measurements,
class RandomWalkKalmanFilter2D (line 206) | class RandomWalkKalmanFilter2D(KalmanFilter2D): #TODO UPDATE THIS DOCUME...
method __init__ (line 222) | def __init__(self, pos_weight, velocity_weight, std_process, std_measu...
FILE: paper_experiments/utils/kf_3d.py
class KalmanFilter3D (line 8) | class KalmanFilter3D(EKF.EKF):
method __init__ (line 24) | def __init__(self):
method initiate (line 76) | def initiate(self, measurement):
method motion_update (line 103) | def motion_update(self, mean, covariance):
method get_motion_cov (line 110) | def get_motion_cov(self, mean, covariance):
method sensor_update (line 115) | def sensor_update(self, mean, covariance):
method get_innovation_cov (line 122) | def get_innovation_cov(self, mean, covariance):
method adjust_angle (line 126) | def adjust_angle(self, measured, target):
method update (line 131) | def update(self, mean, covariance, meas_in, marginalization=None, JPDA...
method gating_distance (line 139) | def gating_distance(self, mean, covariance, measurements,
FILE: paper_experiments/utils/linear_assignment.py
function min_marg_matching (line 14) | def min_marg_matching(marginalizations, track_indices=None, max_distance...
function min_cost_matching (line 60) | def min_cost_matching(
function JPDA (line 141) | def JPDA(
function calculate_entropy (line 212) | def calculate_entropy(matrix, idx, idy):
function get_JPDA_output (line 220) | def get_JPDA_output(cluster, cost_matrix, dummy_node_cost_app, dummy_nod...
function matching_cascade (line 273) | def matching_cascade(
function gate_cost_matrix (line 339) | def gate_cost_matrix(
function find_clusters (line 410) | def find_clusters(cost_matrix, cutoff):
FILE: paper_experiments/utils/logger.py
class Logger (line 6) | class Logger:
method __init__ (line 7) | def __init__(self, logdir):
method close (line 12) | def close(self):
method log_scalar (line 15) | def log_scalar(self, tag, value, global_step):
method log_image (line 21) | def log_image(self, tag, img, global_step):
FILE: paper_experiments/utils/mbest_ilp.py
function ilp_assignment (line 16) | def ilp_assignment(model):
function initialize_model (line 29) | def initialize_model(cost_matrix, cutoff, model = None):
function cache (line 59) | def cache(func):
function num_solutions (line 74) | def num_solutions(cost_matrix):
function enumerate_solutions (line 85) | def enumerate_solutions(cost_matrix, cutoff, num_solutions):
function new_m_best_sol (line 104) | def new_m_best_sol(cost_matrix, m_sol, cutoff, model = None):
function linear_assignment_wrapper (line 189) | def linear_assignment_wrapper(a):
FILE: paper_experiments/utils/nn_matching.py
function _pdist (line 6) | def _pdist(a, b):
function _cosine_distance (line 32) | def _cosine_distance(a, b, data_is_normalized=False):
function _cosine_distance_torch (line 57) | def _cosine_distance_torch(a, b, data_is_normalized=False):
function _nn_euclidean_distance (line 66) | def _nn_euclidean_distance(x, y):
function _nn_euclidean_distance_torch (line 86) | def _nn_euclidean_distance_torch(x, y):
function _nn_cosine_distance (line 111) | def _nn_cosine_distance(x, y):
function _nn_cosine_distance_torch (line 131) | def _nn_cosine_distance_torch(x,y):
class NearestNeighborDistanceMetric (line 138) | class NearestNeighborDistanceMetric(object):
method __init__ (line 162) | def __init__(self, metric, budget=None):
method partial_fit (line 178) | def partial_fit(self, features, features_2d, targets, targets_2d, acti...
method distance (line 209) | def distance(self, features, targets, compare_2d=False):
method distance_torch (line 235) | def distance_torch(self, features, targets, compare_2d=False):
method check_samples (line 248) | def check_samples(self, targets):
FILE: paper_experiments/utils/pointnet_tf_util.py
function _variable_on_cpu (line 10) | def _variable_on_cpu(name, shape, initializer, use_fp16=False):
function _variable_with_weight_decay (line 24) | def _variable_with_weight_decay(name, shape, stddev, wd, use_xavier=True):
function conv1d (line 52) | def conv1d(inputs,
function conv2d (line 112) | def conv2d(inputs,
function conv2d_transpose (line 172) | def conv2d_transpose(inputs,
function conv3d (line 252) | def conv3d(inputs,
function fully_connected (line 311) | def fully_connected(inputs,
function max_pool2d (line 350) | def max_pool2d(inputs,
function avg_pool2d (line 375) | def avg_pool2d(inputs,
function max_pool3d (line 401) | def max_pool3d(inputs,
function avg_pool3d (line 426) | def avg_pool3d(inputs,
function batch_norm_template (line 455) | def batch_norm_template(inputs, is_training, scope, moments_dims, bn_dec...
function batch_norm_for_fc (line 495) | def batch_norm_for_fc(inputs, is_training, bn_decay, scope):
function batch_norm_for_conv1d (line 509) | def batch_norm_for_conv1d(inputs, is_training, bn_decay, scope):
function batch_norm_for_conv2d (line 525) | def batch_norm_for_conv2d(inputs, is_training, bn_decay, scope):
function batch_norm_for_conv3d (line 540) | def batch_norm_for_conv3d(inputs, is_training, bn_decay, scope):
function dropout (line 554) | def dropout(inputs,
FILE: paper_experiments/utils/pointnet_transform_nets.py
function input_transform_net (line 10) | def input_transform_net(point_cloud, is_training, bn_decay=None, K=3):
function feature_transform_net (line 54) | def feature_transform_net(inputs, is_training, bn_decay=None, K=64):
FILE: paper_experiments/utils/read_detections.py
function read_ground_truth_2d_detections (line 5) | def read_ground_truth_2d_detections(detection_path_2d, frame_idx, detect...
function read_ground_truth_3d_detections (line 82) | def read_ground_truth_3d_detections(detection_path_3d, frame_idx):
FILE: paper_experiments/utils/resnet_reid_utils.py
class Feature_ResNet (line 14) | class Feature_ResNet(nn.Module):
method __init__ (line 15) | def __init__(self,n_layer,output_color):
method forward (line 24) | def forward(self,x):
class ResNet_Loader (line 35) | class ResNet_Loader(object):
method __init__ (line 36) | def __init__(self,model_path,n_layer=50,batch_size=4,output_color=False):
method inference (line 56) | def inference(self,patches):
FILE: paper_experiments/utils/test_kf/run_kf_test.py
class Track (line 12) | class Track:
method __init__ (line 13) | def __init__(self, track_id, first_detection, kf_type):
method update (line 36) | def update(self, measurement, gt, frame):
method plot (line 53) | def plot(self):
function file2data (line 99) | def file2data(fname):
function cmp_tracks (line 108) | def cmp_tracks(track1, track2):
function cmp (line 114) | def cmp(data, val):
function validate (line 128) | def validate(data, fname):
function run_kf_test (line 137) | def run_kf_test(fname, kf_type):
FILE: paper_experiments/utils/test_kf/write_kf_test.py
function data2file (line 8) | def data2file(data, fname):
function add_noise (line 17) | def add_noise(center, std):
function single_track_4state_test (line 23) | def single_track_4state_test():
function two_track_4state_test (line 49) | def two_track_4state_test():
function single_track_6state_test (line 88) | def single_track_6state_test():
FILE: paper_experiments/utils/track.py
class TrackState (line 9) | class TrackState:
class Track (line 24) | class Track:
method __init__ (line 70) | def __init__(self, mean, covariance, model_probabilities, track_id, n_...
method to_tlwh (line 107) | def to_tlwh(self, kf):
method to_tlbr (line 129) | def to_tlbr(self):
method update_feature (line 143) | def update_feature(self, img, appearance_model):
method predict (line 168) | def predict(self, kf):
method update (line 186) | def update(self, kf, detection, detections_3d=None,
method delete_track (line 239) | def delete_track(self):
method mark_missed (line 242) | def mark_missed(self):
method update_velocity (line 250) | def update_velocity(self, new_detection):
method is_tentative (line 265) | def is_tentative(self):
method is_confirmed (line 270) | def is_confirmed(self):
method is_deleted (line 274) | def is_deleted(self):
method is_exiting (line 278) | def is_exiting(self):
method mark_exiting (line 281) | def mark_exiting(self):
method feature_update (line 284) | def feature_update(self, detections, detection_idx, lstm, JPDA=False, ...
FILE: paper_experiments/utils/track_3d.py
class TrackState (line 6) | class TrackState:
class Track_3d (line 21) | class Track_3d:
method __init__ (line 67) | def __init__(self, mean, covariance, track_id, n_init, max_age,
method to_tlwh3d (line 96) | def to_tlwh3d(self):
method to_tlwh (line 109) | def to_tlwh(self, kf):
method predict (line 124) | def predict(self, kf):
method update (line 139) | def update(self, kf, detection, compare_2d=False,
method mark_missed (line 178) | def mark_missed(self):
method is_tentative (line 186) | def is_tentative(self):
method is_confirmed (line 191) | def is_confirmed(self):
method is_deleted (line 195) | def is_deleted(self):
FILE: paper_experiments/utils/tracker.py
class Tracker (line 16) | class Tracker:
method __init__ (line 46) | def __init__(self, max_age=5, n_init=3,
method gated_metric (line 81) | def gated_metric(self, tracks, dets, track_indices, detection_indices,...
method predict (line 100) | def predict(self):
method update (line 109) | def update(self, cur_frame, detections, compare_2d = False):
method _match (line 170) | def _match(self, detections, compare_2d):
method _initiate_track (line 207) | def _initiate_track(self, detection, flow=None):
method prune_tracks (line 221) | def prune_tracks(self):
FILE: paper_experiments/utils/tracker_3d.py
class Tracker_3d (line 14) | class Tracker_3d:
method __init__ (line 44) | def __init__(self, max_age=30, n_init=3,
method gated_metric (line 68) | def gated_metric(self, tracks, dets, track_indices, detection_indices,...
method predict (line 97) | def predict(self):
method update (line 106) | def update(self, input_img, detections, compare_2d):
method _match (line 159) | def _match(self, detections, compare_2d):
method _initiate_track (line 195) | def _initiate_track(self, detection):
method prune_tracks (line 205) | def prune_tracks(self):
FILE: paper_experiments/utils/tracking_utils.py
function create_detector (line 14) | def create_detector(config_path, weight_path, cuda):
function get_depth_patches (line 23) | def get_depth_patches(point_cloud, box_3d, ids_3d, rot_angles, num_point...
function non_max_suppression_3D_prime (line 63) | def non_max_suppression_3D_prime(detections, boxes_3d, ids_3d, ids_2d, n...
function non_max_suppression_3D (line 77) | def non_max_suppression_3D(depth_patches, ids_3d, ids_2d, nms_thresh = 1...
function iou_3d (line 108) | def iou_3d(patch_1, patch_2):
function convert_detections (line 128) | def convert_detections(detections, features, appearance_features, detect...
function combine_features (line 147) | def combine_features(features, depth_features, ids_3d, combination_model...
function filter (line 164) | def filter(detections):
FILE: paper_experiments/utils/visualise.py
function draw_track (line 9) | def draw_track(bbox, track = None, bbox_colors = None, det = True,
function draw_detection (line 38) | def draw_detection(detection, color='k'):
function draw_ellipse (line 47) | def draw_ellipse(track, color):
function draw_velocity (line 73) | def draw_velocity(track, color):
function draw_box3d (line 84) | def draw_box3d(mu, color, alpha, facecolor='none', ax=None):
function draw_velocity_3d (line 105) | def draw_velocity_3d(track, color, ax=None):
function draw_ellipse3d (line 116) | def draw_ellipse3d(covariance, x, y, color, ax=None):
function draw_track3d (line 136) | def draw_track3d(track, color, ax=None):
function draw_detection3d (line 148) | def draw_detection3d(det, color, ax=None):
FILE: paper_experiments/utils/yolo_utils/datasets.py
class ImageFolder (line 19) | class ImageFolder(Dataset):
method __init__ (line 20) | def __init__(self, folder_path, img_size=416):
method __getitem__ (line 24) | def __getitem__(self, index):
method __len__ (line 45) | def __len__(self):
class ListDataset (line 49) | class ListDataset(Dataset):
method __init__ (line 50) | def __init__(self, list_path, img_size=416):
method __getitem__ (line 57) | def __getitem__(self, index):
method __len__ (line 120) | def __len__(self):
FILE: paper_experiments/utils/yolo_utils/parse_config.py
function parse_model_config (line 3) | def parse_model_config(path):
function parse_data_config (line 23) | def parse_data_config(path):
FILE: paper_experiments/utils/yolo_utils/utils.py
function load_classes (line 14) | def load_classes(path):
function weights_init_normal (line 23) | def weights_init_normal(m):
function compute_ap (line 32) | def compute_ap(recall, precision):
function bbox_iou (line 60) | def bbox_iou(box1, box2, x1y1x2y2=True):
function bbox_iou_numpy (line 93) | def bbox_iou_numpy(box1, box2):
function non_max_suppression (line 127) | def non_max_suppression(prediction, num_classes, conf_thres=0.5, nms_thr...
function build_targets (line 187) | def build_targets(
function to_categorical (line 256) | def to_categorical(y, num_classes):
FILE: src/3d_detector.py
class Detector_3d (line 28) | class Detector_3d:
method __init__ (line 29) | def __init__(self):
method get_3d_feature (line 60) | def get_3d_feature(self, y1_bboxes, pointcloud_upper, pointcloud_lower):
method publish_pointcloud_from_array (line 169) | def publish_pointcloud_from_array(self, pointcloud, publisher, frame =...
method cleanup (line 178) | def cleanup(self):
function main (line 181) | def main(args):
FILE: src/EKF.py
function squared_mahalanobis_distance (line 79) | def squared_mahalanobis_distance(mean, covariance, measurements):
class EKF (line 95) | class EKF(object):
method __init__ (line 101) | def __init__(self):
method initiate (line 104) | def initiate(self, measurement):
method predict_mean (line 121) | def predict_mean(self, mean):
method get_process_noise (line 127) | def get_process_noise(self, mean, covariance):
method predict_covariance (line 130) | def predict_covariance(self, mean, covariance):
method project_mean (line 133) | def project_mean(self, mean):
method project_cov (line 138) | def project_cov(self, mean, covariance):
method predict (line 141) | def predict(self, mean, covariance):
method get_innovation_cov (line 165) | def get_innovation_cov(self, covariance):
method project (line 168) | def project(self, mean, covariance):
method update (line 189) | def update(self, mean, covariance, measurement_t, marginalization=None...
FILE: src/JPDA_matching.py
function get_unmatched (line 8) | def get_unmatched(all_idx, matches, i, marginalization=None):
class Matcher (line 19) | class Matcher:
method __init__ (line 21) | def __init__(self, detections, marginalizations, confirmed_tracks,
method match (line 31) | def match(self):
method get_matches (line 37) | def get_matches(self):
method get_unmatched_tracks (line 51) | def get_unmatched_tracks(self):
method get_unmatched_detections (line 55) | def get_unmatched_detections(self):
method max_match (line 58) | def max_match(self):
method max_and_threshold_matching (line 84) | def max_and_threshold_matching(self):
method hungarian (line 103) | def hungarian(self):
FILE: src/aligned_reid_model.py
class Model (line 10) | class Model(nn.Module):
method __init__ (line 11) | def __init__(self, local_conv_out_channels=128, num_classes=None):
method forward (line 24) | def forward(self, x):
function conv3x3 (line 55) | def conv3x3(in_planes, out_planes, stride=1):
class BasicBlock (line 61) | class BasicBlock(nn.Module):
method __init__ (line 64) | def __init__(self, inplanes, planes, stride=1, downsample=None):
method forward (line 74) | def forward(self, x):
class Bottleneck (line 93) | class Bottleneck(nn.Module):
method __init__ (line 96) | def __init__(self, inplanes, planes, stride=1, downsample=None):
method forward (line 109) | def forward(self, x):
class ResNet (line 132) | class ResNet(nn.Module):
method __init__ (line 134) | def __init__(self, block, layers):
method _make_layer (line 155) | def _make_layer(self, block, planes, blocks, stride=1):
method forward (line 172) | def forward(self, x):
function remove_fc (line 186) | def remove_fc(state_dict):
function resnet18 (line 195) | def resnet18(pretrained=False):
function resnet34 (line 207) | def resnet34(pretrained=False):
function resnet50 (line 219) | def resnet50(pretrained=False):
function resnet101 (line 231) | def resnet101(pretrained=False):
function resnet152 (line 244) | def resnet152(pretrained=False):
FILE: src/aligned_reid_utils.py
function time_str (line 16) | def time_str(fmt=None):
function load_pickle (line 22) | def load_pickle(path):
function save_pickle (line 34) | def save_pickle(obj, path):
function save_mat (line 41) | def save_mat(ndarray, path):
function to_scalar (line 46) | def to_scalar(vt):
function transfer_optim_state (line 57) | def transfer_optim_state(state, device_id=-1):
function may_transfer_optims (line 82) | def may_transfer_optims(optims, device_id=-1):
function may_transfer_modules_optims (line 95) | def may_transfer_modules_optims(modules_and_or_optims, device_id=-1):
class TransferVarTensor (line 114) | class TransferVarTensor(object):
method __init__ (line 117) | def __init__(self, device_id=-1):
method __call__ (line 120) | def __call__(self, var_or_tensor):
class TransferModulesOptims (line 125) | class TransferModulesOptims(object):
method __init__ (line 128) | def __init__(self, device_id=-1):
method __call__ (line 131) | def __call__(self, modules_and_or_optims):
function set_devices (line 135) | def set_devices(sys_device_ids):
function set_devices_for_ml (line 164) | def set_devices_for_ml(sys_device_ids):
function load_ckpt (line 229) | def load_ckpt(modules_optims, ckpt_file, load_to_cpu=True, verbose=True):
function save_ckpt (line 252) | def save_ckpt(modules_optims, ep, scores, ckpt_file):
function load_state_dict (line 273) | def load_state_dict(model, src_state_dict):
function is_iterable (line 314) | def is_iterable(obj):
function may_set_mode (line 318) | def may_set_mode(maybe_modules, mode):
function may_make_dir (line 331) | def may_make_dir(path):
class AverageMeter (line 347) | class AverageMeter(object):
method __init__ (line 351) | def __init__(self):
method reset (line 357) | def reset(self):
method update (line 363) | def update(self, val, n=1):
class RunningAverageMeter (line 370) | class RunningAverageMeter(object):
method __init__ (line 373) | def __init__(self, hist=0.99):
method reset (line 378) | def reset(self):
method update (line 382) | def update(self, val):
class RecentAverageMeter (line 390) | class RecentAverageMeter(object):
method __init__ (line 393) | def __init__(self, hist_size=100):
method reset (line 398) | def reset(self):
method update (line 402) | def update(self, val):
method avg (line 409) | def avg(self):
function get_model_wrapper (line 414) | def get_model_wrapper(model, multi_gpu):
class ReDirectSTD (line 422) | class ReDirectSTD(object):
method __init__ (line 442) | def __init__(self, fpath=None, console='stdout', immediately_visible=F...
method __del__ (line 463) | def __del__(self):
method __enter__ (line 466) | def __enter__(self):
method __exit__ (line 469) | def __exit__(self, *args):
method write (line 472) | def write(self, msg):
method flush (line 484) | def flush(self):
method close (line 491) | def close(self):
function set_seed (line 497) | def set_seed(seed):
function print_array (line 514) | def print_array(array, fmt='{:.2f}', end=' '):
function str2bool (line 525) | def str2bool(v):
function tight_float_str (line 529) | def tight_float_str(x, fmt='{:.4f}'):
function find_index (line 533) | def find_index(seq, item):
function adjust_lr_exp (line 540) | def adjust_lr_exp(optimizer, base_lr, ep, total_ep, start_decay_at_ep):
function adjust_lr_staircase (line 572) | def adjust_lr_staircase(optimizer, base_lr, ep, decay_at_epochs, factor):
function measure_time (line 608) | def measure_time(enter_msg):
function generate_features (line 615) | def generate_features(appearance_model, patches):
function generate_features_batched (line 626) | def generate_features_batched(appearance_model, patches, object_ids = No...
function get_image_patches (line 674) | def get_image_patches(input_img, detections):
function create_appearance_model (line 689) | def create_appearance_model(alignreid_checkpoint, resnet_reid_checkpoint...
FILE: src/calibration.py
class Calibration (line 8) | class Calibration(object):
method __init__ (line 33) | def __init__(self, calib_filepath):
method read_calib_file (line 71) | def read_calib_file(self, filepath):
method read_calib_from_video (line 92) | def read_calib_from_video(self, calib_root_dir):
method cart2hom (line 107) | def cart2hom(self, pts_3d):
method cart2hom_torch (line 115) | def cart2hom_torch(self, pts_3d):
method project_velo_to_ref (line 123) | def project_velo_to_ref(self, pts_3d_velo):
method project_ref_to_velo (line 127) | def project_ref_to_velo(self, pts_3d_ref):
method project_rect_to_ref (line 131) | def project_rect_to_ref(self, pts_3d_rect):
method project_ref_to_rect (line 135) | def project_ref_to_rect(self, pts_3d_ref):
method project_ref_to_rect_torch (line 139) | def project_ref_to_rect_torch(self, pts_3d_ref):
method project_rect_to_velo (line 143) | def project_rect_to_velo(self, pts_3d_rect):
method project_velo_to_rect (line 150) | def project_velo_to_rect(self, pts_3d_velo):
method project_rect_to_image (line 157) | def project_rect_to_image(self, pts_3d_rect):
method project_rect_to_image_torch (line 167) | def project_rect_to_image_torch(self, pts_3d_rect):
method project_ref_to_image_torch (line 177) | def project_ref_to_image_torch(self, pts_3d_ref):
method project_velo_to_image (line 187) | def project_velo_to_image(self, pts_3d_velo):
method project_image_to_rect (line 197) | def project_image_to_rect(self, uv_depth):
method project_image_to_velo (line 211) | def project_image_to_velo(self, uv_depth):
function rotx (line 215) | def rotx(t):
function roty (line 224) | def roty(t):
function rotz (line 233) | def rotz(t):
function transform_from_rot_trans (line 242) | def transform_from_rot_trans(R, t):
function inverse_rigid_trans (line 249) | def inverse_rigid_trans(Tr):
function read_label (line 258) | def read_label(label_filename):
function load_image (line 263) | def load_image(img_filename):
function load_velo_scan (line 266) | def load_velo_scan(velo_filename):
function project_to_image (line 271) | def project_to_image(pts_3d, P):
function compute_box_3d (line 291) | def compute_box_3d(obj, P):
function compute_orientation_3d (line 329) | def compute_orientation_3d(obj, P):
function draw_projected_box3d (line 358) | def draw_projected_box3d(image, qs, color=(255,255,255), thickness=2):
class OmniCalibration (line 385) | class OmniCalibration(Calibration):
method __init__ (line 386) | def __init__(self, calib_folder):
method project_ref_to_image_torch (line 402) | def project_ref_to_image_torch(self, pointcloud):
method project_image_to_rect (line 413) | def project_image_to_rect(self, uvdepth):
method project_velo_to_ref (line 422) | def project_velo_to_ref(self, pointcloud):
method move_lidar_to_camera_frame (line 430) | def move_lidar_to_camera_frame(self, pointcloud, upper = True):
method calculate_median_param_value (line 450) | def calculate_median_param_value(self, param):
FILE: src/combination_model.py
class CombiNet (line 6) | class CombiNet(nn.Module):
method __init__ (line 7) | def __init__(self, in_dim = 2560, hidden_units = 512, out_dim = 2560):
method forward (line 16) | def forward(self, x):
class CombiLSTM (line 30) | class CombiLSTM(nn.Module):
method __init__ (line 31) | def __init__(self, in_dim = 2560, hidden_units = 512, out_dim = 2560):
method forward (line 43) | def forward(self, x, hidden = None):
function weight_init (line 65) | def weight_init(m):
FILE: src/deep_sort_utils.py
function non_max_suppression (line 6) | def non_max_suppression(boxes, max_bbox_overlap, scores=None):
FILE: src/detection.py
class Detection (line 5) | class Detection(object):
method __init__ (line 29) | def __init__(self, tlwh, box_3d, confidence, appearance_feature, featu...
method to_tlbr (line 44) | def to_tlbr(self):
method to_xyah (line 52) | def to_xyah(self):
method to_xywh (line 60) | def to_xywh(self):
method get_3d_distance (line 67) | def get_3d_distance(self):
FILE: src/distances.py
function norm2squared_matrix (line 10) | def norm2squared_matrix(objs, hyps, max_d2=float('inf')):
function iou_matrix (line 52) | def iou_matrix(objs, hyps, max_iou=1.):
function find_area (line 112) | def find_area(vertices):
function get_angle (line 118) | def get_angle(p):
function clip_polygon (line 125) | def clip_polygon(box1, box2):
function compute_intersection_point (line 163) | def compute_intersection_point(pt1, pt2, line1):
function point_inside_edge (line 189) | def point_inside_edge(pt, edge):
function iou_matrix_3d (line 197) | def iou_matrix_3d(objs, hyps, max_iou=1.):
FILE: src/double_measurement_kf.py
class KF_3D (line 18) | class KF_3D(kf_2d.KalmanFilter2D):
method __init__ (line 33) | def __init__(self, calib, pos_weight_3d, pos_weight, velocity_weight, ...
method initiate (line 64) | def initiate(self, measurement_3d):
method get_process_noise (line 83) | def get_process_noise(self, mean):
method get_2d_measurement_noise (line 102) | def get_2d_measurement_noise(self, measurement_2d):
method get_3d_measurement_noise (line 114) | def get_3d_measurement_noise(self, measurement):
method gating_distance (line 130) | def gating_distance(self, mean, covariance, measurements,
method project_cov (line 182) | def project_cov(self, mean, covariance):
method project_cov_2d (line 191) | def project_cov_2d(self, mean, covariance, H_2d):
method update (line 200) | def update(self, mean, covariance, measurement_2d, measurement_3d = No...
method get_2d_measurement_matrix (line 312) | def get_2d_measurement_matrix(self, mean, corner_points, corner_points...
method jacobian (line 349) | def jacobian(self, pt_3d):
method jacobian_omni (line 356) | def jacobian_omni(self, pt_3d):
method calculate_corners (line 370) | def calculate_corners(self, box):
method corner_jacobian (line 389) | def corner_jacobian(self, pt_3d, corner_idx):
method project_2d (line 416) | def project_2d(self, pts_3d):
function swap (line 436) | def swap(detections_3d, iou, idx, swap_prob = 0):
FILE: src/evaluation/distances 2.py
function norm2squared_matrix (line 10) | def norm2squared_matrix(objs, hyps, max_d2=float('inf')):
function iou_matrix (line 52) | def iou_matrix(objs, hyps, max_iou=1.):
function find_area (line 112) | def find_area(vertices):
function get_angle (line 118) | def get_angle(p):
function clip_polygon (line 125) | def clip_polygon(box1, box2):
function sort_points (line 163) | def sort_points(pts, center):
function compute_intersection_point (line 169) | def compute_intersection_point(pt1, pt2, line1):
function point_inside_edge (line 195) | def point_inside_edge(pt, edge):
function iou_matrix_3d (line 203) | def iou_matrix_3d(objs, hyps, max_iou=1.):
FILE: src/evaluation/distances.py
function norm2squared_matrix (line 10) | def norm2squared_matrix(objs, hyps, max_d2=float('inf')):
function iou_matrix (line 52) | def iou_matrix(objs, hyps, max_iou=1.):
function find_area (line 112) | def find_area(vertices):
function get_angle (line 118) | def get_angle(p):
function clip_polygon (line 125) | def clip_polygon(box1, box2):
function sort_points (line 163) | def sort_points(pts, center):
function compute_intersection_point (line 169) | def compute_intersection_point(pt1, pt2, line1):
function point_inside_edge (line 195) | def point_inside_edge(pt, edge):
function iou_matrix_3d (line 203) | def iou_matrix_3d(objs, hyps, max_iou=1.):
FILE: src/featurepointnet_model.py
class FPointNet (line 13) | class FPointNet():
method __init__ (line 14) | def __init__(self, config_path):
method __call__ (line 57) | def __call__(self, input_point_cloud, rot_angle, peds=False):
method get_instance_seg_v1_net (line 126) | def get_instance_seg_v1_net(self, point_cloud, one_hot_vec, is_trainin...
method get_3d_box_estimation_v1_net (line 197) | def get_3d_box_estimation_v1_net(self, object_point_cloud, one_hot_vec...
method get_model (line 246) | def get_model(self, point_cloud, one_hot_vec, is_training, bn_decay=No...
method get_depth_feature_op (line 294) | def get_depth_feature_op(self, is_training):
method get_depth_feature (line 317) | def get_depth_feature(self, object_pointcloud):
method softmax (line 323) | def softmax(self, x):
function create_depth_model (line 330) | def create_depth_model(model, config_path):
FILE: src/featurepointnet_model_util.py
function tf_gather_object_pc (line 39) | def tf_gather_object_pc(point_cloud, mask, npoints=512):
function get_box3d_corners_helper (line 72) | def get_box3d_corners_helper(centers, headings, sizes):
function get_box3d_corners (line 99) | def get_box3d_corners(center, heading_residuals, size_residuals):
function huber_loss (line 124) | def huber_loss(error, delta):
function parse_output_to_tensors (line 132) | def parse_output_to_tensors(output, end_points):
function from_prediction_to_label_format (line 170) | def from_prediction_to_label_format(center, angle_class, angle_res,\
function size2class (line 179) | def size2class(size, type_name):
function class2size (line 194) | def class2size(pred_cls, residual):
function angle2class (line 199) | def angle2class(angle, num_class):
function class2angle (line 219) | def class2angle(pred_cls, residual, num_class, to_label_format=True):
function rotate_pc_along_y (line 230) | def rotate_pc_along_y(pc, rot_angle):
function rotate_pc_along_y_torch (line 247) | def rotate_pc_along_y_torch(pc, rot_angle):
function placeholder_inputs (line 270) | def placeholder_inputs(batch_size, num_point):
function point_cloud_masking (line 295) | def point_cloud_masking(point_cloud, logits, end_points, xyz_only=True):
function get_center_regression_net (line 343) | def get_center_regression_net(object_point_cloud, one_hot_vec,
function softmax (line 380) | def softmax(x):
function get_loss (line 388) | def get_loss(mask_label, center_label, \
function get_lidar_in_image_fov (line 510) | def get_lidar_in_image_fov(pc_velo, calib, xmin, ymin, xmax, ymax,
function preprocess_pointcloud (line 524) | def preprocess_pointcloud(detections, point_cloud, pc_image_coord,
function generate_detections_3d (line 576) | def generate_detections_3d(detector, detections_2d, point_cloud, calib, ...
function convert_depth_features (line 590) | def convert_depth_features(depth_features_orig, ids_3d):
FILE: src/featurepointnet_tf_util.py
function _variable_on_cpu (line 10) | def _variable_on_cpu(name, shape, initializer, use_fp16=False):
function _variable_with_weight_decay (line 24) | def _variable_with_weight_decay(name, shape, stddev, wd, use_xavier=True):
function conv1d (line 52) | def conv1d(inputs,
function conv2d (line 120) | def conv2d(inputs,
function conv2d_transpose (line 188) | def conv2d_transpose(inputs,
function conv3d (line 268) | def conv3d(inputs,
function fully_connected (line 327) | def fully_connected(inputs,
function max_pool2d (line 366) | def max_pool2d(inputs,
function avg_pool2d (line 391) | def avg_pool2d(inputs,
function max_pool3d (line 417) | def max_pool3d(inputs,
function avg_pool3d (line 442) | def avg_pool3d(inputs,
function batch_norm_template_unused (line 468) | def batch_norm_template_unused(inputs, is_training, scope, moments_dims,...
function batch_norm_template (line 512) | def batch_norm_template(inputs, is_training, scope, moments_dims_unused,...
function batch_norm_for_fc (line 534) | def batch_norm_for_fc(inputs, is_training, bn_decay, scope):
function batch_norm_for_conv1d (line 548) | def batch_norm_for_conv1d(inputs, is_training, bn_decay, scope, data_for...
function batch_norm_for_conv2d (line 565) | def batch_norm_for_conv2d(inputs, is_training, bn_decay, scope, data_for...
function batch_norm_for_conv3d (line 580) | def batch_norm_for_conv3d(inputs, is_training, bn_decay, scope):
function dropout (line 594) | def dropout(inputs,
FILE: src/iou_matching.py
function iou (line 8) | def iou(bbox, candidates):
function iou_cost (line 43) | def iou_cost(tracks, detections, track_indices=None,
FILE: src/kf_2d.py
class KalmanFilter2D (line 8) | class KalmanFilter2D(EKF.EKF):
method __init__ (line 25) | def __init__(self, pos_weight, velocity_weight, std_process, std_measu...
method initiate (line 47) | def initiate(self, measurement, flow):
method predict_mean (line 87) | def predict_mean(self, mean):
method predict_covariance (line 93) | def predict_covariance(self, mean, covariance):
method get_process_noise (line 100) | def get_process_noise(self, mean):
method project_mean (line 118) | def project_mean(self, mean):
method get_measurement_noise (line 126) | def get_measurement_noise(self, measurement):
method project_cov (line 138) | def project_cov(self, mean, covariance):
method gating_distance (line 147) | def gating_distance(self, mean, covariance, measurements,
class RandomWalkKalmanFilter2D (line 187) | class RandomWalkKalmanFilter2D(KalmanFilter2D): #TODO UPDATE THIS DOCUME...
method __init__ (line 203) | def __init__(self, pos_weight, velocity_weight, std_process, std_measu...
FILE: src/linear_assignment.py
function min_marg_matching (line 15) | def min_marg_matching(marginalizations, track_indices=None, max_distance...
function min_cost_matching (line 61) | def min_cost_matching(
function JPDA (line 142) | def JPDA(
function calculate_entropy (line 208) | def calculate_entropy(matrix, idx, idy):
function get_JPDA_output (line 216) | def get_JPDA_output(cluster, cost_matrix, dummy_node_cost_app, dummy_nod...
function matching_cascade (line 273) | def matching_cascade(
function gate_cost_matrix (line 339) | def gate_cost_matrix(
function find_clusters (line 401) | def find_clusters(cost_matrix, cutoff):
FILE: src/mbest_ilp.py
function ilp_assignment (line 16) | def ilp_assignment(model):
function initialize_model (line 29) | def initialize_model(cost_matrix, cutoff, model = None):
function cache (line 59) | def cache(func):
function num_solutions (line 74) | def num_solutions(cost_matrix):
function enumerate_solutions (line 85) | def enumerate_solutions(cost_matrix, cutoff, num_solutions):
function new_m_best_sol (line 104) | def new_m_best_sol(cost_matrix, m_sol, cutoff, model = None):
function linear_assignment_wrapper (line 189) | def linear_assignment_wrapper(a):
FILE: src/nn_matching.py
function _pdist (line 6) | def _pdist(a, b):
function _cosine_distance (line 32) | def _cosine_distance(a, b, data_is_normalized=False):
function _cosine_distance_torch (line 57) | def _cosine_distance_torch(a, b, data_is_normalized=False):
function _nn_euclidean_distance (line 66) | def _nn_euclidean_distance(x, y):
function _nn_euclidean_distance_torch (line 86) | def _nn_euclidean_distance_torch(x, y):
function _nn_cosine_distance (line 111) | def _nn_cosine_distance(x, y):
function _nn_cosine_distance_torch (line 131) | def _nn_cosine_distance_torch(x,y):
class NearestNeighborDistanceMetric (line 138) | class NearestNeighborDistanceMetric(object):
method __init__ (line 162) | def __init__(self, metric, budget=None):
method partial_fit (line 178) | def partial_fit(self, features, features_2d, targets, targets_2d, acti...
method distance (line 209) | def distance(self, features, targets, compare_2d=False):
method distance_torch (line 235) | def distance_torch(self, features, targets, compare_2d=False):
method check_samples (line 248) | def check_samples(self, targets):
FILE: src/pointnet_model.py
class PointNet (line 9) | class PointNet():
method __init__ (line 10) | def __init__(self, config_path):
method __call__ (line 37) | def __call__(self, input_point_cloud):
method placeholder_inputs (line 43) | def placeholder_inputs(self, batch_size, num_point):
method get_model (line 49) | def get_model(self, point_cloud, is_training, bn_decay=None):
method get_loss (line 96) | def get_loss(self, pred, label, end_points, reg_weight=0.001):
FILE: src/template 2.py
class Appearance_Features (line 19) | class Appearance_Features:
method __init__ (line 20) | def __init__(self):
method get_2d_feature (line 39) | def get_2d_feature(self, y1_bboxes, ros_image):
method cleanup (line 86) | def cleanup(self):
function main (line 89) | def main(args):
FILE: src/template.py
class Appearance_Features (line 19) | class Appearance_Features:
method __init__ (line 20) | def __init__(self):
method get_2d_feature (line 39) | def get_2d_feature(self, y1_bboxes, ros_image):
method cleanup (line 86) | def cleanup(self):
function main (line 89) | def main(args):
FILE: src/track_3d 2.py
class TrackState (line 6) | class TrackState:
class Track_3d (line 21) | class Track_3d:
method __init__ (line 67) | def __init__(self, mean, covariance, track_id, n_init, max_age,
method to_tlwh3d (line 97) | def to_tlwh3d(self):
method to_tlwh (line 112) | def to_tlwh(self, kf):
method predict (line 127) | def predict(self, kf):
method update (line 142) | def update(self, kf, detection, compare_2d=False,
method mark_missed (line 191) | def mark_missed(self):
method is_tentative (line 199) | def is_tentative(self):
method is_confirmed (line 204) | def is_confirmed(self):
method is_deleted (line 208) | def is_deleted(self):
method feature_update (line 212) | def feature_update(self, detections, detection_idx, lstm, JPDA=False, ...
method get_cov (line 244) | def get_cov(self):
FILE: src/track_3d.py
class TrackState (line 6) | class TrackState:
class Track_3d (line 21) | class Track_3d:
method __init__ (line 67) | def __init__(self, mean, covariance, track_id, n_init, max_age,
method to_tlwh3d (line 97) | def to_tlwh3d(self):
method to_tlwh (line 112) | def to_tlwh(self, kf):
method predict (line 127) | def predict(self, kf):
method update (line 142) | def update(self, kf, detection, compare_2d=False,
method mark_missed (line 191) | def mark_missed(self):
method is_tentative (line 199) | def is_tentative(self):
method is_confirmed (line 204) | def is_confirmed(self):
method is_deleted (line 208) | def is_deleted(self):
method feature_update (line 212) | def feature_update(self, detections, detection_idx, lstm, JPDA=False, ...
method get_cov (line 244) | def get_cov(self):
FILE: src/tracker_3d 2.py
class Tracker_3d (line 15) | class Tracker_3d:
method __init__ (line 45) | def __init__(self, max_age=30, n_init=3,
method gated_metric (line 71) | def gated_metric(self, tracks, dets, track_indices, detection_indices,...
method predict (line 99) | def predict(self):
method update (line 108) | def update(self, input_img, detections):
method _match (line 148) | def _match(self, detections):
method _initiate_track (line 213) | def _initiate_track(self, detection):
method prune_tracks (line 223) | def prune_tracks(self):
FILE: src/tracker_3d.py
class Tracker_3d (line 15) | class Tracker_3d:
method __init__ (line 45) | def __init__(self, max_age=30, n_init=3,
method gated_metric (line 71) | def gated_metric(self, tracks, dets, track_indices, detection_indices,...
method predict (line 99) | def predict(self):
method update (line 108) | def update(self, input_img, detections):
method _match (line 148) | def _match(self, detections):
method _initiate_track (line 213) | def _initiate_track(self, detection):
method prune_tracks (line 223) | def prune_tracks(self):
FILE: src/tracker_3d_node 2.py
class Tracker_3D_node (line 30) | class Tracker_3D_node:
method __init__ (line 31) | def __init__(self):
method do_3d_tracking (line 87) | def do_3d_tracking(self, detections_2d, detections_3d):
method find_time_diff_2d (line 157) | def find_time_diff_2d(self, a):
method find_time_diff_3d (line 161) | def find_time_diff_3d(self, a):
method cleanup (line 165) | def cleanup(self):
function main (line 174) | def main(args):
FILE: src/tracker_3d_node.py
class Tracker_3D_node (line 30) | class Tracker_3D_node:
method __init__ (line 31) | def __init__(self):
method do_3d_tracking (line 87) | def do_3d_tracking(self, detections_2d, detections_3d):
method find_time_diff_2d (line 157) | def find_time_diff_2d(self, a):
method find_time_diff_3d (line 161) | def find_time_diff_3d(self, a):
method cleanup (line 165) | def cleanup(self):
function main (line 174) | def main(args):
FILE: src/tracking_utils 2.py
function create_detector (line 13) | def create_detector(config_path, weight_path, cuda):
function get_depth_patches (line 22) | def get_depth_patches(point_cloud, box_3d, ids_3d, rot_angles, num_point...
function non_max_suppression_3D_prime (line 62) | def non_max_suppression_3D_prime(detections, boxes_3d, ids_3d, ids_2d, n...
function non_max_suppression_3D (line 76) | def non_max_suppression_3D(depth_patches, ids_3d, ids_2d, nms_thresh = 1...
function iou_3d (line 107) | def iou_3d(patch_1, patch_2):
function convert_detections (line 127) | def convert_detections(detections, features, appearance_features, detect...
function combine_features (line 146) | def combine_features(features, depth_features, ids_3d, combination_model...
function filter (line 164) | def filter(detections):
FILE: src/tracking_utils.py
function create_detector (line 13) | def create_detector(config_path, weight_path, cuda):
function get_depth_patches (line 22) | def get_depth_patches(point_cloud, box_3d, ids_3d, rot_angles, num_point...
function non_max_suppression_3D_prime (line 62) | def non_max_suppression_3D_prime(detections, boxes_3d, ids_3d, ids_2d, n...
function non_max_suppression_3D (line 76) | def non_max_suppression_3D(depth_patches, ids_3d, ids_2d, nms_thresh = 1...
function iou_3d (line 107) | def iou_3d(patch_1, patch_2):
function convert_detections (line 127) | def convert_detections(detections, features, appearance_features, detect...
function combine_features (line 146) | def combine_features(features, depth_features, ids_3d, combination_model...
function filter (line 164) | def filter(detections):
Condensed preview — 100 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (761K chars).
[
{
"path": "CMakeLists.txt",
"chars": 6977,
"preview": "cmake_minimum_required(VERSION 2.8.3)\nproject(jpda_rospack)\n\n## Compile as C++11, supported in ROS Kinetic and newer\n# a"
},
{
"path": "LICENSE",
"chars": 1091,
"preview": "MIT License\n\nCopyright (c) 2020 Stanford Vision and Learning Group\n\nPermission is hereby granted, free of charge, to any"
},
{
"path": "README.md",
"chars": 3833,
"preview": "# JRMOT ROS package\n\nThe repository contains the code for the work \"JRMOT: A Real-Time 3D Multi-Object Tracker and a New"
},
{
"path": "calib/cameras.yaml",
"chars": 3432,
"preview": "stitching:\n radius: 3360000\n rotation: 0\n scalewidth: 1831\n crop: 1\ncameras:\n # camera order matters!\n sensor_0:\n "
},
{
"path": "calib/defaults.yaml",
"chars": 877,
"preview": "calibrated:\n # the lidar_to_rgb parameters allow tweaking of the transformation between lidar and rgb frames\n # the de"
},
{
"path": "config/featurepointnet.cfg",
"chars": 113,
"preview": "[general]\nnum_point = 1024\nmodel_path = /home/sibot/jr2_catkin_ws/src/jpda_rospack/src/fpointnet_jrdb/model.ckpt\n"
},
{
"path": "launch/jpda_tracker.launch",
"chars": 1457,
"preview": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n\n<launch>\n <!-- Console launch prefix -->\n <arg name=\"output\" default=\"s"
},
{
"path": "msg/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "msg/detection2d_with_feature.msg",
"chars": 711,
"preview": "# This message contains a 2D bounding box corresponding to the detection of a person\n# Also contains the feature of this"
},
{
"path": "msg/detection2d_with_feature_array.msg",
"chars": 66,
"preview": "Header header\ndetection2d_with_feature[] detection2d_with_features"
},
{
"path": "msg/detection3d_with_feature.msg",
"chars": 879,
"preview": "# This message contains a 3D bounding box corresponding to the detection of a person\n# Also contains the feature of this"
},
{
"path": "msg/detection3d_with_feature_array.msg",
"chars": 66,
"preview": "Header header\ndetection3d_with_feature[] detection3d_with_features"
},
{
"path": "package.xml",
"chars": 3010,
"preview": "<?xml version=\"1.0\"?>\n<package format=\"2\">\n <name>jpda_rospack</name>\n <version>0.0.1</version>\n <description>The jpd"
},
{
"path": "paper_experiments/models/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "paper_experiments/models/aligned_reid_model.py",
"chars": 7458,
"preview": "import torch\nimport torch.nn as nn\nimport torch.nn.init as init\nimport torch.nn.functional as F\nimport torch.utils.model"
},
{
"path": "paper_experiments/models/combination_model.py",
"chars": 2103,
"preview": "import pdb\n\nimport numpy as np\nimport torch.nn as nn\n\nclass CombiNet(nn.Module):\n\tdef __init__(self, in_dim = 2560, hidd"
},
{
"path": "paper_experiments/models/deep_sort_model.py",
"chars": 1424,
"preview": "import tensorflow as tf\nfrom skimage.transform import resize\nimport numpy as np\n\nclass ImageEncoder(object):\n\n def __"
},
{
"path": "paper_experiments/models/featurepointnet_model.py",
"chars": 16765,
"preview": "import os, pdb\nimport numpy as np\nimport tensorflow as tf\ntf.logging.set_verbosity(tf.logging.ERROR)\nimport configparser"
},
{
"path": "paper_experiments/models/pointnet_model.py",
"chars": 4887,
"preview": "import os, pdb\nimport tensorflow as tf\ntf.logging.set_verbosity(tf.logging.ERROR)\nimport configparser\nfrom utils.pointne"
},
{
"path": "paper_experiments/models/resnet_reid_models.py",
"chars": 4430,
"preview": "import torch\nimport torch.nn as nn\nfrom torch.autograd import Variable\nimport torchvision.models as models\nfrom torchvis"
},
{
"path": "paper_experiments/models/yolo_models.py",
"chars": 14786,
"preview": "from __future__ import division\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.autograd "
},
{
"path": "paper_experiments/requirements.txt",
"chars": 1760,
"preview": "absl-py==0.7.0\nastor==0.7.1\nbackcall==0.1.0\nbleach==3.3.0\ncatkin-pkg==0.4.12\ncertifi==2018.11.29\nchardet==3.0.4\ncloudpic"
},
{
"path": "paper_experiments/track.py",
"chars": 19764,
"preview": "import open3d as o3d\nimport torch\nimport argparse\nimport os, pdb, sys, copy, pickle\nimport time\nimport random\nimport num"
},
{
"path": "paper_experiments/utils/EKF.py",
"chars": 7392,
"preview": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\nimport scipy.linalg\nimport pdb\n\n\"\"\"\nTable for the 0.95 quantile of the chi"
},
{
"path": "paper_experiments/utils/JPDA_matching.py",
"chars": 4046,
"preview": "# vim: expandtab:ts=4:sw=4\nfrom __future__ import absolute_import\nimport numpy as np\nfrom linear_assignment import min_m"
},
{
"path": "paper_experiments/utils/aligned_reid_utils.py",
"chars": 23425,
"preview": "from __future__ import print_function\nimport os\nimport os.path as osp\nimport pickle\nfrom scipy import io\nimport datetime"
},
{
"path": "paper_experiments/utils/assign_ids_detections.py",
"chars": 2124,
"preview": "import numpy as np\nimport os\nimport pdb\nfrom tqdm import tqdm\nfrom deep_sort_utils import non_max_suppression as deepsor"
},
{
"path": "paper_experiments/utils/calibration.py",
"chars": 16997,
"preview": "import numpy as np\nimport cv2\nimport os\nimport yaml\nimport torch\nimport pdb\n\nclass Calibration(object):\n ''' Calibrat"
},
{
"path": "paper_experiments/utils/combine_and_process_detections.py",
"chars": 3126,
"preview": "import os\nfrom os import listdir\nfrom os.path import isfile, join\n\n#root = \"/cvgl2/u/mihirp/depth_tracking/data/JRDB/seq"
},
{
"path": "paper_experiments/utils/dataset.py",
"chars": 8101,
"preview": "import glob\nimport os\nimport pdb\nimport random\nimport sys\nfrom itertools import compress\n\nimport numpy as np\nimport torc"
},
{
"path": "paper_experiments/utils/deep_sort_utils.py",
"chars": 2197,
"preview": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\nimport cv2\n\n\ndef non_max_suppression(boxes, max_bbox_overlap, scores=None)"
},
{
"path": "paper_experiments/utils/detection.py",
"chars": 2273,
"preview": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\n\n\nclass Detection(object):\n \"\"\"\n This class represents a bounding bo"
},
{
"path": "paper_experiments/utils/double_measurement_kf.py",
"chars": 23706,
"preview": "import random\nimport numpy as np\nimport scipy.linalg\nimport EKF\nimport pdb\nimport kf_2d\nimport os\nimport pickle\nimport t"
},
{
"path": "paper_experiments/utils/evaluate_detections.py",
"chars": 4760,
"preview": "import numpy as np\nimport os\nimport pdb\nfrom tqdm import tqdm\nfrom deep_sort_utils import non_max_suppression as deepsor"
},
{
"path": "paper_experiments/utils/featurepointnet_model_util.py",
"chars": 25485,
"preview": "import open3d as o3d\nimport numpy as np\nimport tensorflow as tf\nimport os\nimport sys\nimport torch\nBASE_DIR = os.path.dir"
},
{
"path": "paper_experiments/utils/featurepointnet_tf_util.py",
"chars": 21762,
"preview": "\"\"\" Wrapper functions for TensorFlow layers.\n\nAuthor: Charles R. Qi\nDate: November 2017\n\"\"\"\n\nimport numpy as np\nimport t"
},
{
"path": "paper_experiments/utils/imm.py",
"chars": 12814,
"preview": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\nimport scipy.linalg\nimport utils.EKF as EKF\nimport pdb\nimport utils.kf_2d "
},
{
"path": "paper_experiments/utils/iou_matching.py",
"chars": 3372,
"preview": "# vim: expandtab:ts=4:sw=4\nfrom __future__ import absolute_import\nimport numpy as np\nfrom . import linear_assignment\nimp"
},
{
"path": "paper_experiments/utils/kf_2d.py",
"chars": 10355,
"preview": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\nimport scipy.linalg\nimport utils.EKF as EKF\nimport pdb\nnp.set_printoptions"
},
{
"path": "paper_experiments/utils/kf_3d.py",
"chars": 6181,
"preview": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\nimport scipy.linalg\nimport EKF\nimport pdb\n\n\nclass KalmanFilter3D(EKF.EKF):"
},
{
"path": "paper_experiments/utils/linear_assignment.py",
"chars": 18699,
"preview": "# vim: expandtab:ts=4:sw=4\nfrom __future__ import absolute_import\nimport numpy as np\nfrom sklearn.utils.linear_assignmen"
},
{
"path": "paper_experiments/utils/logger.py",
"chars": 1134,
"preview": "import io, os\nimport numpy as np\nfrom PIL import Image\nimport tensorflow as tf\n#courtesy https://becominghuman.ai/loggin"
},
{
"path": "paper_experiments/utils/mbest_ilp.py",
"chars": 9392,
"preview": "from gurobipy import Model, quicksum, LinExpr, GRB\nimport numpy as np\nimport copy\nimport time\nfrom sklearn.utils.linear_"
},
{
"path": "paper_experiments/utils/nn_matching.py",
"chars": 8291,
"preview": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\nimport pdb\nimport torch\n\ndef _pdist(a, b):\n \"\"\"Compute pair-wise square"
},
{
"path": "paper_experiments/utils/pointnet_tf_util.py",
"chars": 19388,
"preview": "\"\"\" Wrapper functions for TensorFlow layers.\n\nAuthor: Charles R. Qi\nDate: November 2016\n\"\"\"\n\nimport numpy as np\nimport t"
},
{
"path": "paper_experiments/utils/pointnet_transform_nets.py",
"chars": 4297,
"preview": "import tensorflow as tf\nimport numpy as np\nimport sys\nimport os\nBASE_DIR = os.path.dirname(os.path.abspath(__file__))\nsy"
},
{
"path": "paper_experiments/utils/read_detections.py",
"chars": 4678,
"preview": "import numpy as np\nimport pdb\nfrom deep_sort_utils import non_max_suppression as deepsort_nms\n\ndef read_ground_truth_2d_"
},
{
"path": "paper_experiments/utils/resnet_reid_utils.py",
"chars": 4006,
"preview": "import torch\nimport os\nimport sys\nimport torch.nn as nn\nfrom torch.autograd import Variable\nfrom torchvision import tran"
},
{
"path": "paper_experiments/utils/test_jpda.py",
"chars": 1095,
"preview": "from gurobipy import *\nfrom numpy import *\n'''\ndef mycallback(model, where):\n if where == GRB.callback.MIP:\n p"
},
{
"path": "paper_experiments/utils/test_kf/.gitignore",
"chars": 4,
"preview": "*.p\n"
},
{
"path": "paper_experiments/utils/test_kf/run_kf_test.py",
"chars": 5003,
"preview": "import sys\nsys.path.insert(0, '..')\nimport kalman_filter\nimport kf_simple3d\nimport numpy as np\nimport matplotlib.pyplot "
},
{
"path": "paper_experiments/utils/test_kf/write_kf_test.py",
"chars": 2583,
"preview": "import sys\nsys.path.insert(0, '..')\nimport kalman_filter\nimport numpy as np\nimport pickle\nimport pdb\n\ndef data2file(data"
},
{
"path": "paper_experiments/utils/track.py",
"chars": 12377,
"preview": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\nimport pdb\nimport torch\nimport copy\n\nfrom .imm import IMMFilter2D\n\nclass T"
},
{
"path": "paper_experiments/utils/track_3d.py",
"chars": 6442,
"preview": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\nimport pdb\nimport torch\n\nclass TrackState:\n \"\"\"\n Enumeration type fo"
},
{
"path": "paper_experiments/utils/tracker.py",
"chars": 11193,
"preview": "# vim: expandtab:ts=4:sw=4\nfrom __future__ import absolute_import\nimport numpy as np\nimport pdb\nfrom . import kf_2d, kf_"
},
{
"path": "paper_experiments/utils/tracker_3d.py",
"chars": 9781,
"preview": "# vim: expandtab:ts=4:sw=4\nfrom __future__ import absolute_import\nimport numpy as np\nimport pdb\nfrom . import double_mea"
},
{
"path": "paper_experiments/utils/tracking_utils.py",
"chars": 6888,
"preview": "import torch, sys, os, pdb\nimport numpy as np\nfrom PIL import Image\nfrom scipy.spatial import Delaunay\nsys.path.append(o"
},
{
"path": "paper_experiments/utils/visualise.py",
"chars": 5072,
"preview": "import matplotlib\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as plt_patches\nimport numpy as np\nimport uti"
},
{
"path": "paper_experiments/utils/yolo_utils/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "paper_experiments/utils/yolo_utils/datasets.py",
"chars": 4241,
"preview": "import glob\nimport random\nimport os\nimport numpy as np\n\nimport torch\n\nfrom torch.utils.data import Dataset\nfrom PIL impo"
},
{
"path": "paper_experiments/utils/yolo_utils/parse_config.py",
"chars": 1261,
"preview": "\n\ndef parse_model_config(path):\n \"\"\"Parses the yolo-v3 layer configuration file and returns module definitions\"\"\"\n "
},
{
"path": "paper_experiments/utils/yolo_utils/utils.py",
"chars": 9955,
"preview": "from __future__ import division\nimport math\nimport time\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as"
},
{
"path": "requirements.txt",
"chars": 154,
"preview": "cycler==0.10.0\nkiwisolver==1.1.0\nmatplotlib==3.1.2\nnumpy==1.21.0\nopencv-python==4.2.0.32\npyparsing==2.4.6\npython-dateuti"
},
{
"path": "src/3d_detector.py",
"chars": 8994,
"preview": "#!/home/sibot/anaconda2/bin/python\n\"\"\" yolo_bbox_to_sort.py\n Subscribe to the Yolo 2 bboxes, and publish the detectio"
},
{
"path": "src/EKF.py",
"chars": 6920,
"preview": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\nimport scipy.linalg\nimport pdb\n\n\"\"\"\nTable for the 0.95 quantile of the chi"
},
{
"path": "src/JPDA_matching.py",
"chars": 4019,
"preview": "# vim: expandtab:ts=4:sw=4\nfrom __future__ import absolute_import\nimport numpy as np\nfrom linear_assignment import min_m"
},
{
"path": "src/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "src/aligned_reid_model.py",
"chars": 7345,
"preview": "import torch\nimport torch.nn as nn\nimport torch.nn.init as init\nimport torch.nn.functional as F\nimport torch.utils.model"
},
{
"path": "src/aligned_reid_utils.py",
"chars": 20940,
"preview": "from __future__ import print_function\nimport os\nimport os.path as osp\nimport pickle\nfrom scipy import io\nimport datetime"
},
{
"path": "src/calibration.py",
"chars": 17365,
"preview": "import numpy as np\nimport cv2\nimport os\nimport yaml\nimport torch\nimport pdb\n\nclass Calibration(object):\n ''' Calibrat"
},
{
"path": "src/combination_model.py",
"chars": 2101,
"preview": "import pdb\n\nimport numpy as np\nimport torch.nn as nn\n\nclass CombiNet(nn.Module):\n\tdef __init__(self, in_dim = 2560, hidd"
},
{
"path": "src/deep_sort_utils.py",
"chars": 1994,
"preview": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\nimport cv2\n\n\ndef non_max_suppression(boxes, max_bbox_overlap, scores=None)"
},
{
"path": "src/detection.py",
"chars": 2211,
"preview": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\n\n\nclass Detection(object):\n \"\"\"\n This class represents a bounding bo"
},
{
"path": "src/distances.py",
"chars": 8172,
"preview": "\"\"\"py-motmetrics - metrics for multiple object tracker (MOT) benchmarking.\n\nChristoph Heindl, 2017\nhttps://github.com/ch"
},
{
"path": "src/double_measurement_kf.py",
"chars": 19217,
"preview": "import random\nimport numpy as np\nimport scipy.linalg\nimport EKF\nimport pdb\nimport kf_2d\nimport os\nimport pickle\nimport t"
},
{
"path": "src/evaluation/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "src/evaluation/distances 2.py",
"chars": 8430,
"preview": "\"\"\"py-motmetrics - metrics for multiple object tracker (MOT) benchmarking.\n\nChristoph Heindl, 2017\nhttps://github.com/ch"
},
{
"path": "src/evaluation/distances.py",
"chars": 8430,
"preview": "\"\"\"py-motmetrics - metrics for multiple object tracker (MOT) benchmarking.\n\nChristoph Heindl, 2017\nhttps://github.com/ch"
},
{
"path": "src/featurepointnet_model.py",
"chars": 16504,
"preview": "import os, pdb\nimport numpy as np\nimport tensorflow as tf\ntf.logging.set_verbosity(tf.logging.ERROR)\nimport configparser"
},
{
"path": "src/featurepointnet_model_util.py",
"chars": 26104,
"preview": "# import open3d as o3d\nimport numpy as np\nimport tensorflow as tf\nimport os\nimport sys\nimport torch\nBASE_DIR = os.path.d"
},
{
"path": "src/featurepointnet_tf_util.py",
"chars": 21762,
"preview": "\"\"\" Wrapper functions for TensorFlow layers.\n\nAuthor: Charles R. Qi\nDate: November 2017\n\"\"\"\n\nimport numpy as np\nimport t"
},
{
"path": "src/iou_matching.py",
"chars": 3365,
"preview": "# vim: expandtab:ts=4:sw=4\nfrom __future__ import absolute_import\nimport numpy as np\nimport linear_assignment\nimport pdb"
},
{
"path": "src/kf_2d.py",
"chars": 9616,
"preview": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\nimport scipy.linalg\nimport EKF\nimport pdb\nnp.set_printoptions(precision=4,"
},
{
"path": "src/linear_assignment.py",
"chars": 18297,
"preview": "# vim: expandtab:ts=4:sw=4\nfrom __future__ import absolute_import\nimport numpy as np\nfrom sklearn.utils.linear_assignmen"
},
{
"path": "src/mbest_ilp.py",
"chars": 9392,
"preview": "from gurobipy import Model, quicksum, LinExpr, GRB\nimport numpy as np\nimport copy\nimport time\nfrom sklearn.utils.linear_"
},
{
"path": "src/nn_matching.py",
"chars": 8284,
"preview": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\nimport pdb\nimport torch\n\ndef _pdist(a, b):\n \"\"\"Compute pair-wise square"
},
{
"path": "src/pointnet_model.py",
"chars": 4887,
"preview": "import os, pdb\nimport tensorflow as tf\ntf.logging.set_verbosity(tf.logging.ERROR)\nimport configparser\nfrom utils.pointne"
},
{
"path": "src/template 2.py",
"chars": 4061,
"preview": "#!/home/sibot/anaconda2/bin/python\n\"\"\" yolo_bbox_to_sort.py\n Subscribe to the Yolo 2 bboxes, and publish the detectio"
},
{
"path": "src/template.py",
"chars": 4061,
"preview": "#!/home/sibot/anaconda2/bin/python\n\"\"\" yolo_bbox_to_sort.py\n Subscribe to the Yolo 2 bboxes, and publish the detectio"
},
{
"path": "src/track_3d 2.py",
"chars": 9356,
"preview": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\nimport pdb\nimport torch\n\nclass TrackState:\n \"\"\"\n Enumeration type fo"
},
{
"path": "src/track_3d.py",
"chars": 9356,
"preview": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\nimport pdb\nimport torch\n\nclass TrackState:\n \"\"\"\n Enumeration type fo"
},
{
"path": "src/tracker_3d 2.py",
"chars": 11233,
"preview": "# vim: expandtab:ts=4:sw=4\nfrom __future__ import absolute_import\nimport numpy as np\nimport pdb\nimport double_measuremen"
},
{
"path": "src/tracker_3d.py",
"chars": 11233,
"preview": "# vim: expandtab:ts=4:sw=4\nfrom __future__ import absolute_import\nimport numpy as np\nimport pdb\nimport double_measuremen"
},
{
"path": "src/tracker_3d_node 2.py",
"chars": 7568,
"preview": "#!/home/sibot/anaconda2/bin/python\n\"\"\" yolo_bbox_to_sort.py\n Subscribe to the Yolo 2 bboxes, and publish the detectio"
},
{
"path": "src/tracker_3d_node.py",
"chars": 7568,
"preview": "#!/home/sibot/anaconda2/bin/python\n\"\"\" yolo_bbox_to_sort.py\n Subscribe to the Yolo 2 bboxes, and publish the detectio"
},
{
"path": "src/tracking_utils 2.py",
"chars": 6892,
"preview": "import torch, sys, os, pdb\nimport numpy as np\nfrom PIL import Image\nfrom scipy.spatial import Delaunay\nsys.path.append(o"
},
{
"path": "src/tracking_utils.py",
"chars": 6892,
"preview": "import torch, sys, os, pdb\nimport numpy as np\nfrom PIL import Image\nfrom scipy.spatial import Delaunay\nsys.path.append(o"
}
]
// ... and 3 more files (download for full content)
About this extraction
This page contains the full source code of the StanfordVL/JRMOT_ROS GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 100 files (715.3 KB), approximately 190.5k tokens, and a symbol index with 839 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.